/prog/scrape.NET

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.IO;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System.Data.SQLite;

namespace Progscrape
{
    class Progscrape
    {
        static void Main(string[] args)
        {
            var db = new SQLiteConnection("Data Source=prog.db");
            db.Open();
            (new SQLiteCommand("CREATE TABLE IF NOT EXISTS posts (thread INTEGER, id INTEGER, author TEXT, email TEXT, trip TEXT, time INTEGER, body TEXT, primary key (thread, id));", db)).ExecuteNonQuery();
            (new SQLiteCommand("CREATE TABLE IF NOT EXISTS threads (thread INTEGER PRIMARY KEY, title TEXT, last_post INTEGER);", db)).ExecuteNonQuery();
            var subjectTxtReq = (HttpWebRequest)WebRequest.Create("http://dis.4chan.org/prog/subject.txt");
            subjectTxtReq.UserAgent = "progscrape.NET/1.0";
            subjectTxtReq.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
            subjectTxtReq.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
            Console.Write("Fetching subject.txt...");
            var subjectTxt = new StreamReader(subjectTxtReq.GetResponse().GetResponseStream(), new UTF8Encoding()).ReadToEnd().Split(new Char[] {'\n'});
            Console.WriteLine("done.");
            foreach (var line in subjectTxt)
            {
                var data = line.Split(new string[] { "<>" }, StringSplitOptions.None);
                if(data.Length == 7)
                {
                    var thread = int.Parse(data[3]);
                    var getLastPost = new SQLiteCommand("SELECT last_post FROM threads WHERE thread = @thread;", db);
                    getLastPost.Parameters.Add(new SQLiteParameter("thread", thread));
                    var lastPost = getLastPost.ExecuteScalar();
                    if (lastPost == null)
                    {
                        lastPost = 0;
                        var insertThread = new SQLiteCommand("INSERT INTO threads VALUES(@thread, @title, @lastPost);", db);
                        insertThread.Parameters.Add(new SQLiteParameter("thread", thread));
                        insertThread.Parameters.Add(new SQLiteParameter("title", data[0]));
                        insertThread.Parameters.Add(new SQLiteParameter("lastPost", (int)lastPost));
                        insertThread.ExecuteNonQuery();
                    }
                       
                    if (int.Parse(data[6]) > (int)lastPost)
                    {
                        lastPost = int.Parse(data[6]);
                        var updateThread = new SQLiteCommand("UPDATE threads SET last_post = @lastPost;", db);
                        updateThread.Parameters.Add(new SQLiteParameter("lastPost", lastPost));
                        updateThread.ExecuteNonQuery();
                        Console.Write("updating thread {0}...", thread);
                        var threadReq = (HttpWebRequest)WebRequest.Create("http://dis.4chan.org/json/prog/" + data[3]);
                        threadReq.UserAgent = "progscrape.NET/1.0";
                        threadReq.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
                        threadReq.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
                        var threadJson = JObject.Parse(new StreamReader(threadReq.GetResponse().GetResponseStream(), new UTF8Encoding()).ReadToEnd());
                        for (int i = 0; i < int.Parse(data[4]); ++i)
                        {
                            var postJson = threadJson[i.ToString()];
                            if (postJson != null)
                            {
                                var updatePost = new SQLiteCommand("INSERT OR REPLACE INTO posts VALUES(@thread, @id, @author, \"\", \"\", @time, @body);", db);
                                updatePost.Parameters.Add(new SQLiteParameter("thread", thread));
                                updatePost.Parameters.Add(new SQLiteParameter("id", i));
                                updatePost.Parameters.Add(new SQLiteParameter("author", (string)postJson["Author"]));
                                updatePost.Parameters.Add(new SQLiteParameter("time", int.Parse((string)postJson["now"])));
                                updatePost.Parameters.Add(new SQLiteParameter("body", (string)postJson["com"]));
                                updatePost.ExecuteNonQuery();
                            }
                        }
                        Console.WriteLine("done.");
                    }
                }
                else
                    Console.WriteLine("subject.txt fail (length {0}, skipping thread): {1}", data.Length, line);
            }
            db.Close();
        }
    }
}

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.IO;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System.Data.SQLite;

namespace Progscrape
{
    class Progscrape
    {
        static void Main(string[] args)
        {
            var db = new SQLiteConnection("Data Source=prog.db");
            db.Open();
            (new SQLiteCommand("CREATE TABLE IF NOT EXISTS posts (thread INTEGER, id INTEGER, author TEXT, email TEXT, trip TEXT, time INTEGER, body TEXT, primary key (thread, id));", db)).ExecuteNonQuery();
            (new SQLiteCommand("CREATE TABLE IF NOT EXISTS threads (thread INTEGER PRIMARY KEY, title TEXT, last_post INTEGER);", db)).ExecuteNonQuery();
            var retry = true;
            string[] subjectTxt = {};
            Console.Write("Fetching subject.txt...");
            while (retry)
                try
                {
                    var subjectTxtReq = (HttpWebRequest)WebRequest.Create("http://dis.4chan.org/prog/subject.txt");
                    subjectTxtReq.UserAgent = "progscrape.NET/1.0";
                    subjectTxtReq.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
                    subjectTxtReq.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
                    subjectTxt = new StreamReader(subjectTxtReq.GetResponse().GetResponseStream(), new UTF8Encoding()).ReadToEnd().Split(new Char[] { '\n' });
                    retry = false;
                }
                catch(WebException e)
                {
                    if (e.Status == WebExceptionStatus.Timeout)
                        Console.Write("timed out.\nretrying...");
                    else
                    {
                        Console.WriteLine("error: {0}", e.Message);
                        return;
                    }
                }
            Console.WriteLine("done.");
            foreach (var line in subjectTxt)
            {
                var data = line.Split(new string[] { "<>" }, StringSplitOptions.None);
                if(data.Length == 7)
                {
                    var thread = long.Parse(data[3]);
                    var getLastPost = new SQLiteCommand("SELECT last_post FROM threads WHERE thread = @thread;", db);
                    getLastPost.Parameters.Add(new SQLiteParameter("thread", thread));
                    var lastPost = (long?)getLastPost.ExecuteScalar();
                    if (lastPost == null)
                    {
                        lastPost = 0;
                        var insertThread = new SQLiteCommand("INSERT INTO threads VALUES(@thread, @title, @lastPost);", db);
                        insertThread.Parameters.Add(new SQLiteParameter("thread", thread));
                        insertThread.Parameters.Add(new SQLiteParameter("title", data[0]));
                        insertThread.Parameters.Add(new SQLiteParameter("lastPost", lastPost));
                        insertThread.ExecuteNonQuery();
                    }
                       
                    if (long.Parse(data[6]) > lastPost)
                    {
                        lastPost = long.Parse(data[6]);
                        Console.Write("updating thread {0}...", thread);
                        JObject threadJson = null;
                        retry = true;
                        while(retry)
                            try
                            {
                                var threadReq = (HttpWebRequest)WebRequest.Create("http://dis.4chan.org/json/prog/" + data[3]);
                                threadReq.UserAgent = "progscrape.NET/1.0";
                                threadReq.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
                                threadReq.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
                                threadJson = JObject.Parse(new StreamReader(threadReq.GetResponse().GetResponseStream(), new UTF8Encoding()).ReadToEnd());
                                retry = false;
                            }
                            catch (WebException e)
                            {
                                if (e.Status == WebExceptionStatus.Timeout)
                                    Console.Write("timed out.\nretrying...");
                                else
                                {
                                    Console.WriteLine("error: {0}", e.Message);
                                    return;
                                }
                            }
                        for (int i = 0; i < int.Parse(data[4]); ++i)
                        {
                            var postJson = threadJson[i.ToString()];
                            if (postJson != null)
                            {
                                var updatePost = new SQLiteCommand("INSERT OR REPLACE INTO posts VALUES(@thread, @id, @author, \"\", \"\", @time, @body);", db);
                                updatePost.Parameters.Add(new SQLiteParameter("thread", thread));
                                updatePost.Parameters.Add(new SQLiteParameter("id", i));
                                updatePost.Parameters.Add(new SQLiteParameter("author", (string)postJson["Author"]));
                                updatePost.Parameters.Add(new SQLiteParameter("time", int.Parse((string)postJson["now"])));
                                updatePost.Parameters.Add(new SQLiteParameter("body", (string)postJson["com"]));
                                updatePost.ExecuteNonQuery();
                            }
                        }
                        var updateThread = new SQLiteCommand("UPDATE threads SET last_post = @lastPost;", db);
                        updateThread.Parameters.Add(new SQLiteParameter("lastPost", lastPost));
                        updateThread.ExecuteNonQuery();
                        Console.WriteLine("done.");
                    }
                }
                else
                    Console.WriteLine("subject.txt fail (length {0}, skipping thread): {1}", data.Length, line);
            }
            db.Close();
        }
    }
}

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.IO;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System.Data.SQLite;

namespace Progscrape
{
    class Progscrape
    {
        static void Main(string[] args)
        {
            var db = new SQLiteConnection("Data Source=prog.db");
            db.Open();
            (new SQLiteCommand("CREATE TABLE IF NOT EXISTS posts (thread INTEGER, id INTEGER, author TEXT, email TEXT, trip TEXT, time INTEGER, body TEXT, primary key (thread, id));", db)).ExecuteNonQuery();
            (new SQLiteCommand("CREATE TABLE IF NOT EXISTS threads (thread INTEGER PRIMARY KEY, title TEXT, last_post INTEGER);", db)).ExecuteNonQuery();
            string[] subjectTxt = {};
            Console.Write("Fetching subject.txt...");
            try
            {
                var subjectTxtReq = (HttpWebRequest)WebRequest.Create("http://dis.4chan.org/prog/subject.txt");
                subjectTxtReq.UserAgent = "progscrape.NET/1.0";
                subjectTxtReq.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
                subjectTxtReq.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
                subjectTxt = new StreamReader(subjectTxtReq.GetResponse().GetResponseStream(), new UTF8Encoding()).ReadToEnd().Split(new Char[] { '\n' });
            }
            catch(WebException e)
            {
                Console.WriteLine();
                Console.WriteLine("error: {0}", e.Message);
                db.Close();
                return;
            }
            Console.WriteLine("done.");
            foreach (var line in subjectTxt)
            {
                var data = line.Split(new string[] { "<>" }, StringSplitOptions.None);
                if(data.Length == 7)
                {
                    var thread = long.Parse(data[3]);
                    var getLastPost = new SQLiteCommand("SELECT last_post FROM threads WHERE thread = @thread;", db);
                    getLastPost.Parameters.Add(new SQLiteParameter("thread", thread));
                    var lastPost = (long?)getLastPost.ExecuteScalar();
                    if (lastPost == null)
                    {
                        lastPost = 0;
                        var insertThread = new SQLiteCommand("INSERT INTO threads VALUES(@thread, @title, @lastPost);", db);
                        insertThread.Parameters.Add(new SQLiteParameter("thread", thread));
                        insertThread.Parameters.Add(new SQLiteParameter("title", data[0]));
                        insertThread.Parameters.Add(new SQLiteParameter("lastPost", lastPost));
                        insertThread.ExecuteNonQuery();
                    }
                       
                    if (long.Parse(data[6]) > lastPost)
                    {
                        lastPost = long.Parse(data[6]);
                        Console.Write("updating thread {0}...", thread);
                        JObject threadJson = null;
                        try
                        {
                            var threadReq = (HttpWebRequest)WebRequest.Create("http://dis.4chan.org/json/prog/" + data[3]);
                            threadReq.UserAgent = "progscrape.NET/1.0";
                            threadReq.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
                            threadReq.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
                            threadJson = JObject.Parse(new StreamReader(threadReq.GetResponse().GetResponseStream(), new UTF8Encoding()).ReadToEnd());
                        }
                        catch (WebException e)
                        {
                            Console.WriteLine();
                            Console.WriteLine("error: {0}", e.Message);
                            db.Close();
                            return;
                        }
                        for (int i = 0; i < int.Parse(data[4]); ++i)
                        {
                            var postJson = threadJson[i.ToString()];
                            if (postJson != null)
                            {
                                var updatePost = new SQLiteCommand("INSERT OR REPLACE INTO posts VALUES(@thread, @id, @author, \"\", \"\", @time, @body);", db);
                                updatePost.Parameters.Add(new SQLiteParameter("thread", thread));
                                updatePost.Parameters.Add(new SQLiteParameter("id", i));
                                updatePost.Parameters.Add(new SQLiteParameter("author", (string)postJson["Author"]));
                                updatePost.Parameters.Add(new SQLiteParameter("time", int.Parse((string)postJson["now"])));
                                updatePost.Parameters.Add(new SQLiteParameter("body", (string)postJson["com"]));
                                updatePost.ExecuteNonQuery();
                            }
                        }
                        var updateThread = new SQLiteCommand("UPDATE threads SET last_post = @lastPost;", db);
                        updateThread.Parameters.Add(new SQLiteParameter("lastPost", lastPost));
                        updateThread.ExecuteNonQuery();
                        Console.WriteLine("done.");
                    }
                }
                else
                    Console.WriteLine("subject.txt fail (length {0}, skipping thread): {1}", data.Length, line);
            }
            db.Close();
        }
    }
}

How do I compile this?

>>4
To assemble a codan, we generally use the sharp jagged edge of a bottle broken off at the neck coated in several layers of pages from SICP. This gives the codan structural integrity while also allowing it to absorb a surplus of embalming fluid for use in the procedure. Diagram 14 will indicate appropriate settings on the fellatio burner for applying this type of coating

Since even the smallest amount of contamination can result in the codan sparking, spitting, or pulling out, you'll want to wear your safety goggles at all times. Naturally you should do this anyway, but a disappointingly sizeable minority of people feel like they don't have to adhere to the safety protocols that every one of you signed on your first day here. Keep in mind that the deceptive statistical unlikelihood of a life-threatening accident is just that--deceptive. The original formulation of Murphy's Law should be in chapter 2 of your field manuals, and you're expected to memorize it.

After completing all other pre-anusry setup, but before inserting the codan, take care to ensure that the proglodyte is in a stable and comfortable position, so as to avoid such disastrous consequences as having it pass out from exertion. Place your free hand gently on the buttock to help spread it open. The anus will react initially with a slight tensing, followed by relaxation as the codan eases in. An unusual response at this stage should be taken as a sign of insufficient ink application, and the procedure should be aborted. Otherwise, proceed with complete insertion up to the secondary fill line etched into the codan's surface. At this point the proglodyte should experience an intense yet pleasurable stinging sensation as the liquid epoxy literally melts the flesh of the anus and permanently destroys all the nerve endings in the most painful way imaginable.

Once insertion is complete, proceed with vendor lock-in by depressing lever B of the codan, thus allowing it to rotate about a quarter turn clockwise. When the anus looks sufficiently puckered, pull the codan back out until the fastener catches on the tender and vomit-inducing inside lip of the anus. The tertiary fill line (if present) should once again be visible. Allow lever B to return to its neutral position, securing the codan in place, and depress lever A momentarily to initiate satori transfer. For the duration of the transfer, the proglodyte's increases in programming ability will be indicated by an extra set of shark mouth tattoos appearing spontaneously around the nipples, as well as multiple sets of glowing red eyes on the ends of stalks sprouting out of the nose in place of nose hairs. Completion of the transfer is indicated by the proglodyte swallowing its own feet and entering an infinite loop. As a courtesy to others, please immediately disconnect the codan to avoid turning into Heath Ledger with dyed spiceberry facial hair.

W-why was this posted three times in a row?

>>6
For the different profiles: release, debug, top perforarse

>>6
fixing bugs.

>>4
1. get System.Data.SQLite from http://sqlite.phxsoftware.com/ and install it.
2. get the Json.NET dll from http://james.newtonking.com/pages/json-net.aspx.
3. either compile it like you would any other .NET console application, or just download it from http://hotaru.thinkindifferent.net/progscrape.zip.

Imagine a giant /prog/snake flying towards your mouth, and there's nothing you can do about it. And you're like "Oh man, I'm gonna have to read SICP with this thing", and you brace yourself to read the wizard book. But then, at the last moment, it changes trajectory and hits you in the cudder. You think to yourself "Well, at least I got that out of the way", but then the giant /prog/snake rears back and stabs your cudder again, and again, and again. Eventually, this giant /prog/snake is penetrating your caddeddeddeddeddedder, and you begin to lose control of your motor skills. That's when the giant /prog/snake slaps you across the cheek, causing you to fall out of your chair. Unable to move and at your most vulnerable, the giant /prog/snake finally lodges itself in your anus, where it haxes it uncomfortably for 4, maybe 5 hours.

This is what programming in Scheme feels like.

http://hotaru.thinkindifferent.net/progscrape.zip
now includes all required dlls, and runs on mono if you have sqlite (3.6.1 or newer) installed.

>>10
I bet Xarn doesn't appreciate being linked to in the same section that also includes two Constitution Party websites and Shii.

>>11
actually, Xarn's blog stopped being interesting a long time ago.

Don't change these.

Name:		Email:
Entire Thread Thread List