Split optional post processing tasks out.

Author: musically-ut

README.md

@@ -28,12 +28,15 @@ Schema hints are taken from [a post on Meta.StackExchange](http://meta.stackexch
    - `psql stackoverflow < ./sql/final_post.sql`
    - If you used a different database name, make sure to use that instead of
      `stackoverflow` while executing this step.
+ - For some additional indexes and tables, you can also execute the the following;
+   - `psql stackoverflow < ./sql/optional_post.sql`
+   - Again, remember to user the correct database name here, if not `stackoverflow`.
 
 ## Caveats and TODOs
 
  - It prepares some indexes and views which may not be necessary for your analysis.
- - The `body` field in `Posts` table is NOT populated.
- - The `emailhash` field in `Users` table is NOT populated.
+ - The `Body` field in `Posts` table is NOT populated.
+ - The `EmailHash` field in `Users` table is NOT populated.
  - Some tables (e.g. `PostHistory` and `Comments`) are missing.
 
 ### Sept 2011 data dump

sql/Posts_post.sql

@@ -16,12 +16,3 @@ CREATE INDEX posts_favorite_count_idx ON Posts USING btree (FavoriteCount)
 CREATE INDEX posts_viewcount_idx ON Posts USING btree (ViewCount)
        WITH (FILLFACTOR = 100);
 
--- Composite indexes (optional)
-CREATE INDEX posts_id_post_type_id_idx ON Posts USING btree (Id, PostTypeId)
-       WITH (FILLFACTOR = 100);
-CREATE INDEX posts_id_parent_id_idx ON Posts USING btree (Id, ParentId)
-       WITH (FILLFACTOR = 100);
-CREATE INDEX posts_id_accepted_answers_id_idx ON Posts USING btree (Id, AcceptedAnswerId)
-       WITH (FILLFACTOR = 100);
-CREATE INDEX posts_owner_user_id_creation_date_idx ON Posts USING btree (OwnerUserId, CreationDate)
-       WITH (FILLFACTOR = 100);

sql/final_post.sql

@@ -33,30 +33,6 @@ CREATE INDEX posttags_tagId_idx ON PostTags USING btree (TagId)
        WITH (FILLFACTOR = 100);
 
 
--- UserTagQA TABLE
-DROP TABLE IF EXISTS UserTagQA;
-CREATE TABLE UserTagQA (
-    UserId      int,
-    TagId       int,
-    Questions   int,
-    Answers     int,
-    PRIMARY KEY (UserId, TagId)
-);
-INSERT INTO UserTagQA
-  ( SELECT P.ownerUserId AS UserId,
-           PT.tagId      AS TagId,
-           sum(CASE P.PostTypeId WHEN 1 THEN 1 ELSE 0 END) AS Questions,
-           sum(CASE P.PostTypeId WHEN 2 THEN 1 ELSE 0 END) AS Answers
-    FROM Posts P JOIN PostTags PT ON PT.PostId = P.Id
-    WHERE P.OwnerUserId IS NOT NULL
-    GROUP BY P.OwnerUserId, PT.TagId
-  );
-CREATE INDEX usertagqa_questions_idx ON UserTagQA USING btree (Questions)
-    WITH (FILLFACTOR = 100);
-CREATE INDEX usertagqa_answers_idx ON UserTagQA USING btree (Answers)
-    WITH (FILLFACTOR = 100);
-
-
 -- Tables containing static values
 
 -- CloseAsOffTopicReasonTypes TABLE
@@ -250,23 +226,4 @@ INSERT INTO PostLinkTypes VALUES
   ( 1, 'Linked' ),
   ( 3, 'Duplicate' );
 
--- Questions VIEW
-DROP VIEW IF EXISTS Questions;
-CREATE VIEW Questions AS
-    SELECT Id, AcceptedAnswerId, CreationDate, Score, ViewCount, OwnerUserId,
-           LastEditorUserId, LastEditorDisplayName, LastEditDate,
-           LastActivityDate, Title, Tags, AnswerCount, CommentCount,
-           FavoriteCount, CommunityOwnedDate
-    FROM Posts
-    WHERE PostTypeId = 1;
-
--- Answers VIEW
-DROP VIEW IF EXISTS Answers;
-CREATE VIEW Answers AS
-    SELECT Id, ParentId, CreationDate, Score, OwnerUserId, LastEditorUserId,
-           LastEditorDisplayName, LastEditDate, LastActivityDate,
-           CommentCount, CommunityOwnedDate
-    FROM Posts
-    WHERE PostTypeId = 2;
-
 

sql/optional_post.sql

@@ -0,0 +1,67 @@
+-- These are the optional post processing tasks which may be performed.
+
+-- UserTagQA TABLE
+DROP TABLE IF EXISTS UserTagQA;
+CREATE TABLE UserTagQA (
+    UserId      int,
+    TagId       int,
+    Questions   int,
+    Answers     int,
+    PRIMARY KEY (UserId, TagId)
+);
+INSERT INTO UserTagQA
+  ( SELECT P.ownerUserId AS UserId,
+           PT.tagId      AS TagId,
+           sum(CASE P.PostTypeId WHEN 1 THEN 1 ELSE 0 END) AS Questions,
+           sum(CASE P.PostTypeId WHEN 2 THEN 1 ELSE 0 END) AS Answers
+    FROM Posts P JOIN PostTags PT ON PT.PostId = P.Id
+    WHERE P.OwnerUserId IS NOT NULL
+    GROUP BY P.OwnerUserId, PT.TagId
+  );
+CREATE INDEX usertagqa_questions_idx ON UserTagQA USING btree (Questions)
+    WITH (FILLFACTOR = 100);
+CREATE INDEX usertagqa_answers_idx ON UserTagQA USING btree (Answers)
+    WITH (FILLFACTOR = 100);
+
+
+-- QuestionAnswer TABLE
+DROP TABLE IF EXISTS QuestionAnswer;
+CREATE TABLE QuestionAnswer (
+    QuestionId int,
+    AnswerId   int,
+    PRIMARY KEY (QuestionId, AnswerId)
+);
+INSERT INTO QuestionAnswer
+  ( SELECT P.ParentId as QuestionId, P.Id as AnswerId
+    FROM Posts P WHERE P.PostTypeId = 2
+  );
+
+-- Questions VIEW
+DROP VIEW IF EXISTS Questions;
+CREATE VIEW Questions AS
+    SELECT Id, AcceptedAnswerId, CreationDate, Score, ViewCount, OwnerUserId,
+           LastEditorUserId, LastEditorDisplayName, LastEditDate,
+           LastActivityDate, Title, Tags, AnswerCount, CommentCount,
+           FavoriteCount, CommunityOwnedDate
+    FROM Posts
+    WHERE PostTypeId = 1;
+
+-- Answers VIEW
+DROP VIEW IF EXISTS Answers;
+CREATE VIEW Answers AS
+    SELECT Id, ParentId, CreationDate, Score, OwnerUserId, LastEditorUserId,
+           LastEditorDisplayName, LastEditDate, LastActivityDate,
+           CommentCount, CommunityOwnedDate
+    FROM Posts
+    WHERE PostTypeId = 2;
+
+
+-- Composite indexes for Posts table
+CREATE INDEX posts_id_post_type_id_idx ON Posts USING btree (Id, PostTypeId)
+       WITH (FILLFACTOR = 100);
+CREATE INDEX posts_id_parent_id_idx ON Posts USING btree (Id, ParentId)
+       WITH (FILLFACTOR = 100);
+CREATE INDEX posts_id_accepted_answers_id_idx ON Posts USING btree (Id, AcceptedAnswerId)
+       WITH (FILLFACTOR = 100);
+CREATE INDEX posts_owner_user_id_creation_date_idx ON Posts USING btree (OwnerUserId, CreationDate)
+       WITH (FILLFACTOR = 100);