ENH: Adds PostHistory, Comments and Tags to stackoverflow database

Author: fx86

load_into_pg.py

@@ -74,7 +74,7 @@ def handleTable(table, keys, dbname, mbDbFile, mbHost, mbPort, mbUsername, mbPas
                         if pre != '':
                             cur.execute(pre)
                             conn.commit()
-                        print 'Pre-processing took {} seconds'.format(time.time() - start_time)
+                        print 'Pre-processing took {:.1f} seconds'.format(time.time() - start_time)
 
                         # Handle content of the table
                         start_time = time.time()
@@ -91,7 +91,7 @@ def handleTable(table, keys, dbname, mbDbFile, mbHost, mbPort, mbUsername, mbPas
                                       ' VALUES\n' + valuesStr + ';'
                                 cur.execute(cmd)
                                 conn.commit()
-                        print 'Table processing took {} seconds'.format(time.time() - start_time)
+                        print 'Table processing took {:.1f} seconds'.format(time.time() - start_time)
 
                         # Post-processing (creation of indexes)
                         start_time = time.time()
@@ -119,7 +119,7 @@ def handleTable(table, keys, dbname, mbDbFile, mbHost, mbPort, mbUsername, mbPas
 parser = argparse.ArgumentParser()
 parser.add_argument( 'table'
                    , help    = 'The table to work on.'
-                   , choices = ['Users', 'Badges', 'Posts', 'Tags', 'Votes']
+                   , choices = ['Users', 'Badges', 'Posts', 'Tags', 'Votes', 'PostHistory', 'Comments']
                    )
 
 parser.add_argument( '-d', '--dbname'
@@ -134,12 +134,12 @@ def handleTable(table, keys, dbname, mbDbFile, mbHost, mbPort, mbUsername, mbPas
 
 parser.add_argument( '-u', '--username'
                    , help    = 'Username for the database.'
-                   , default = None
+                   , default = 'postgres'
                    )
 
 parser.add_argument( '-p', '--password'
                    , help    = 'Password for the database.'
-                   , default = None
+                   , default = 'fibinse'
                    )
 
 parser.add_argument( '-P', '--port'
@@ -232,7 +232,25 @@ def handleTable(table, keys, dbname, mbDbFile, mbHost, mbPort, mbUsername, mbPas
       , 'ExcerptPostId'
       , 'WikiPostId'
     ]
-
+elif table == 'PostHistory':
+    keys = [
+        'Id',
+        'PostHistoryTypeId',
+        'PostId',
+        'RevisionGUID',
+        'CreationDate',
+        'UserId',
+        'Text'
+    ]
+elif table == 'Comments':
+    keys = [
+        'Id',
+        'PostId',
+        'Score',
+        'Text',
+        'CreationDate',
+        'UserId',
+    ]
 choice = raw_input('This will drop the {} table. Are you sure [y/n]?'.format(table))
 
 if len(choice) > 0 and choice[0].lower() == 'y':

sql/Comments_post.sql

@@ -0,0 +1,11 @@
+-- hash index takes too long to create
+CREATE INDEX cmnts_post_type_id_idx ON Comments USING btree (Score)
+       WITH (FILLFACTOR = 100);
+CREATE INDEX cmnts_postid_idx ON Comments USING hash (PostId)
+       WITH (FILLFACTOR = 100);
+CREATE INDEX cmnts_revguid_idx ON Comments USING btree (RevisionGUID)
+       WITH (FILLFACTOR = 100);
+CREATE INDEX cmnts_creation_date_idx ON Comments USING btree (CreationDate)
+       WITH (FILLFACTOR = 100);
+CREATE INDEX cmnts_userid_idx ON Comments USING btree (UserId)
+       WITH (FILLFACTOR = 100);

sql/Comments_pre.sql

@@ -0,0 +1,9 @@
+DROP TABLE IF EXISTS Tags CASCADE;
+CREATE TABLE Comments (
+    Id                     int  PRIMARY KEY ,
+    PostId                 int,
+    Score                  int,
+    Post_Text			   text,
+    CreationDate 		   timestamp not NULL ,
+    UserId				   int
+);

sql/PostHistory_post.sql

@@ -0,0 +1,11 @@
+-- hash index takes too long to create
+CREATE INDEX ph_post_type_id_idx ON PostHistory USING btree (PostHistoryTypeId)
+       WITH (FILLFACTOR = 100);
+CREATE INDEX ph_postid_idx ON PostHistory USING hash (PostId)
+       WITH (FILLFACTOR = 100);
+CREATE INDEX ph_revguid_idx ON PostHistory USING btree (RevisionGUID)
+       WITH (FILLFACTOR = 100);
+CREATE INDEX ph_creation_date_idx ON PostHistory USING btree (CreationDate)
+       WITH (FILLFACTOR = 100);
+CREATE INDEX ph_userid_idx ON PostHistory USING btree (UserId)
+       WITH (FILLFACTOR = 100);

sql/PostHistory_pre.sql

@@ -0,0 +1,10 @@
+DROP TABLE IF EXISTS Tags CASCADE;
+CREATE TABLE PostHistory (
+    Id                     int  PRIMARY KEY ,
+    PostHistoryTypeId      int,
+    PostId                 int,
+    RevisionGUID 		   text,
+    CreationDate 		   timestamp not NULL ,
+    UserId				   int,
+    PostText			   text
+);