[ProblemSet] Get problems of databse/shell.

* WIP
* refs #54

Signed-off-by: Eric Wang <skygragon@gmail.com>

Author: skygragon

lib/core.js

@@ -8,6 +8,7 @@ var log = require('loglevel');
 var cache = require('./cache');
 var config = require('./config');
 var client = require('./leetcode_client');
+var queue = require('./queue');
 var h = require('./helper');
 
 var KEY_USER = '.user';
@@ -37,17 +38,38 @@ var core = {};
 core.getProblems = function(cb) {
   var problems = cache.get(KEY_PROBLEMS);
   if (problems) {
-    log.debug('loading from all.json');
+    log.debug('loading from problems.json');
     return cb(null, problems);
   }
 
-  log.debug('running getProblems');
   var user = this.getUser();
-  client.getProblems('algorithms', user, function(e, problems) {
+  var CATEGORIES = ['algorithms', 'database', 'shell'];
+  var KEY_TMP = '.tmp';
+
+  var doTask = function(category, taskDone) {
+    log.debug(category + ': running getProblems');
+    client.getProblems(category, user, function(e, problems) {
+      if (e) {
+        log.debug(category + ': failed to getProblems: ' + e);
+      } else {
+        log.debug(category + ': getProblems got ' +
+            problems.length + ' problems');
+        problems = cache.get(KEY_TMP).concat(problems);
+        cache.set(KEY_TMP, problems);
+      }
+      return taskDone(e);
+    });
+  };
+
+  cache.set(KEY_TMP, []);
+  queue.run(CATEGORIES, doTask, function(e) {
     if (e) return cb(e);
 
     saveUser(user);
+    var problems = cache.get(KEY_TMP);
     cache.set(KEY_PROBLEMS, problems);
+    cache.del(KEY_TMP);
+
     return cb(null, problems);
   });
 };

lib/queue.js

@@ -2,18 +2,20 @@ var config = require('./config');
 
 var queue = {};
 
-function startWorker(ctx) {
+function workerRun(ctx) {
   // no more tasks, quit now
   if (ctx.tasks.length === 0) {
     if (--ctx.workers === 0 && ctx.cb)
-      ctx.cb();
+      ctx.cb(ctx.error);
     return;
   }
 
   var task = ctx.tasks.shift();
   ctx.doTask(task, function(e) {
+    if (e) ctx.error = e;
+
     // TODO: could retry failed task here.
-    startWorker(ctx);
+    setImmediate(workerRun, ctx);
   });
 }
 
@@ -22,11 +24,12 @@ queue.run = function(tasks, doTask, cb) {
     tasks:   tasks,
     doTask:  doTask,
     cb:      cb,
-    workers: config.MAX_WORKERS
+    workers: config.MAX_WORKERS || 1,
+    error:   null
   };
 
   for (var i = 0; i < ctx.workers; ++i) {
-    startWorker(ctx);
+    setImmediate(workerRun, ctx);
   }
 };
 

test/test_core.js

@@ -134,12 +134,16 @@ describe('core', function() {
         cache.del('problems');
 
         client.getProblems = function(category, user, cb) {
-          return cb(null, PROBLEMS);
+          return cb(null, [{category: category}]);
         };
 
         core.getProblems(function(e, problems) {
           assert.equal(e, null);
-          assert.deepEqual(problems, PROBLEMS);
+          assert.deepEqual(problems, [
+            {category: 'algorithms'},
+            {category: 'database'},
+            {category: 'shell'}
+          ]);
           done();
         });
       });