Merge branch 'fx86-master'

Author: musically-ut

.gitignore

@@ -1,3 +1,4 @@
+*.todo
 *.7z
 *.xml
 *.pyc

README.md

@@ -24,6 +24,8 @@ Schema hints are taken from [a post on Meta.StackExchange](http://meta.stackexch
    - `python load_into_pg.py Tags` (not present in earliest dumps)
    - `python load_into_pg.py Users`
    - `python load_into_pg.py Votes`
+   - `python load_into_pg.py PostHistory`
+   - `python load_into_pg.py Comments`
  - Finally, after all the initial tables have been created:
    - `psql stackoverflow < ./sql/final_post.sql`
    - If you used a different database name, make sure to use that instead of

load_into_pg.py

@@ -74,7 +74,7 @@ def handleTable(table, keys, dbname, mbDbFile, mbHost, mbPort, mbUsername, mbPas
                         if pre != '':
                             cur.execute(pre)
                             conn.commit()
-                        print 'Pre-processing took {} seconds'.format(time.time() - start_time)
+                        print 'Pre-processing took {:.1f} seconds'.format(time.time() - start_time)
 
                         # Handle content of the table
                         start_time = time.time()
@@ -91,7 +91,7 @@ def handleTable(table, keys, dbname, mbDbFile, mbHost, mbPort, mbUsername, mbPas
                                       ' VALUES\n' + valuesStr + ';'
                                 cur.execute(cmd)
                                 conn.commit()
-                        print 'Table processing took {} seconds'.format(time.time() - start_time)
+                        print 'Table processing took {:.1f} seconds'.format(time.time() - start_time)
 
                         # Post-processing (creation of indexes)
                         start_time = time.time()
@@ -119,7 +119,7 @@ def handleTable(table, keys, dbname, mbDbFile, mbHost, mbPort, mbUsername, mbPas
 parser = argparse.ArgumentParser()
 parser.add_argument( 'table'
                    , help    = 'The table to work on.'
-                   , choices = ['Users', 'Badges', 'Posts', 'Tags', 'Votes']
+                   , choices = ['Users', 'Badges', 'Posts', 'Tags', 'Votes', 'PostHistory', 'Comments']
                    )
 
 parser.add_argument( '-d', '--dbname'
@@ -232,7 +232,25 @@ def handleTable(table, keys, dbname, mbDbFile, mbHost, mbPort, mbUsername, mbPas
       , 'ExcerptPostId'
       , 'WikiPostId'
     ]
-
+elif table == 'PostHistory':
+    keys = [
+        'Id',
+        'PostHistoryTypeId',
+        'PostId',
+        'RevisionGUID',
+        'CreationDate',
+        'UserId',
+        'Text'
+    ]
+elif table == 'Comments':
+    keys = [
+        'Id',
+        'PostId',
+        'Score',
+        'Text',
+        'CreationDate',
+        'UserId',
+    ]
 choice = raw_input('This will drop the {} table. Are you sure [y/n]?'.format(table))
 
 if len(choice) > 0 and choice[0].lower() == 'y':

sql/Comments_post.sql

@@ -0,0 +1,11 @@
+-- hash index takes too long to create
+CREATE INDEX cmnts_post_type_id_idx ON Comments USING btree (Score)
+       WITH (FILLFACTOR = 100);
+CREATE INDEX cmnts_postid_idx ON Comments USING hash (PostId)
+       WITH (FILLFACTOR = 100);
+CREATE INDEX cmnts_revguid_idx ON Comments USING btree (RevisionGUID)
+       WITH (FILLFACTOR = 100);
+CREATE INDEX cmnts_creation_date_idx ON Comments USING btree (CreationDate)
+       WITH (FILLFACTOR = 100);
+CREATE INDEX cmnts_userid_idx ON Comments USING btree (UserId)
+       WITH (FILLFACTOR = 100);

sql/Comments_pre.sql

@@ -0,0 +1,9 @@
+DROP TABLE IF EXISTS Comments CASCADE;
+CREATE TABLE Comments (
+    Id            int  PRIMARY KEY   ,
+    PostId        int                ,
+    Score         int                ,
+    Post_Text     text               ,
+    CreationDate  timestamp not NULL ,
+    UserId        int
+);

sql/PostHistory_post.sql

@@ -0,0 +1,11 @@
+-- hash index takes too long to create
+CREATE INDEX ph_post_type_id_idx ON PostHistory USING btree (PostHistoryTypeId)
+       WITH (FILLFACTOR = 100);
+CREATE INDEX ph_postid_idx ON PostHistory USING hash (PostId)
+       WITH (FILLFACTOR = 100);
+CREATE INDEX ph_revguid_idx ON PostHistory USING btree (RevisionGUID)
+       WITH (FILLFACTOR = 100);
+CREATE INDEX ph_creation_date_idx ON PostHistory USING btree (CreationDate)
+       WITH (FILLFACTOR = 100);
+CREATE INDEX ph_userid_idx ON PostHistory USING btree (UserId)
+       WITH (FILLFACTOR = 100);

sql/PostHistory_pre.sql

@@ -0,0 +1,10 @@
+DROP TABLE IF EXISTS PostHistory CASCADE;
+CREATE TABLE PostHistory (
+    Id                 int  PRIMARY KEY   ,
+    PostHistoryTypeId  int                ,
+    PostId             int                ,
+    RevisionGUID       text               ,
+    CreationDate       timestamp not NULL ,
+    UserId             int                ,
+    PostText           text
+);