experimentUtils.wppl

// Paths
var fs = webpplFs.node;

var makeRunDir = function(experimentName, runId) {
    return "qaExperiments/" + experimentName + "/run" + runId;
}

var makeRunPath = function(experimentName, runId, dataLabel) {
    var dir = makeRunDir(experimentName, runId);
    var basePath = dir + "/" + experimentName + "-" + runId;
    var basePathWithDataLabel = dataLabel ? basePath + '-' + dataLabel : basePath;
    return { 
        dir: dir,
        dataPath: basePathWithDataLabel + "-data.json",
        paramsPath: basePath + "-params.json",
        logPath: basePath + "-log.csv"
    };
}

var makeQueryPath = function(experimentName, runId, queryId) {
    var runDir = makeRunDir(experimentName, runId);
    var queryDir = runDir + "/query" + queryId;
    var basePath = queryDir + "/" + experimentName + "-" + runId + "-" + queryId;
    return {
        dir: queryDir,
        queryPath: basePath + "-query.json",
        resultPath: basePath + "-results.json",
        expectedKLPath: runDir + "/expected-KL.csv"
    };
}

var makeNetworkTestPath = function(experimentName, runId, networkName) {
    var runDir = makeRunDir(experimentName, runId);
    return runDir + "/nn-" + networkName + ".csv";
}

var makeKLStatPath = function(experimentName, runId) {
    var runDir = makeRunDir(experimentName, runId);
    return {
        prior: runDir + "/expected-KL-prior.csv",
        posterior: runDir + "/expected-KL.csv"
    };
}

var makeNetworkCorrectnessSummary = function(experimentName, runId, networkName) {
    var runDir = makeRunDir(experimentName, runId);
    return runDir + "/nn-" + networkName + "-correctness-summary.csv";
}

var getExperimentNameAndRunId = function() {
    var experimentPath = _.nth(process.argv, -2);
    var experimentName = experimentPath.substring(experimentPath.lastIndexOf('/') + 1, experimentPath.lastIndexOf('.'));
    var runId = _.parseInt(_.nth(process.argv, -1));

    return {
        name: experimentName,
        runId: runId
    }
}

var wordCount = function(utterance, includeFilters) {
    if (includeFilters) {
        return _.words(utterance, RegExp.call(null, includeFilters.join('|'), 'gi')).length;
    }

    return _.words(utterance).length;
}

// Generating data and results
var generateTrainingData = function(listOfLists, fn, numSamples, subsampleList) {
    var recursiveMap = function(listOfLists, fn, numSamples, subsampleList, args /* internal*/) {
        if (listOfLists.length === 0) {
            var resultDist = apply(fn, args);
            return repeat(numSamples, function() {
                return snoc(args, sample(resultDist));
            })
        }

        var samplesFromCurrentList = first(subsampleList);
        var currentList = first(listOfLists);
        var currentListSample = 
            !samplesFromCurrentList 
            ? currentList 
            : _.sampleSize(currentList, samplesFromCurrentList);

        return _.flatten(map(function(e) { 
            recursiveMap(rest(listOfLists), fn, numSamples, rest(subsampleList), snoc(args, e))
        }, currentListSample), /*shallow*/ true)
    };
    
    if (subsampleList) {
        assert.equal(listOfLists.length, subsampleList.length)
    }
    var subsampleListOrDefault = subsampleList || repeat(listOfLists.length, constF(0));
    return recursiveMap(listOfLists, fn, numSamples, subsampleListOrDefault, []);
}

var readOrGenerateTrainingData = function(experimentName, runId, listOfLists, fn, numSamples, dataLabel, subsampleList) {
    var runPath = makeRunPath(experimentName, runId, dataLabel);
    if (fs.existsSync(runPath.dataPath)) {
        return json.read(runPath.dataPath).data;
    }

    if (!fs.existsSync(runPath.dir)) {
        webpplFs.mkdirp(runPath.dir);
    }

    var data = { samples: numSamples, data: generateTrainingData(listOfLists, fn, numSamples, subsampleList) };
    json.write(runPath.dataPath, data);
    return data.data;
}

var computeAveragePriorKL = function(experimentName, runId, groundTruthModel, learnedModel, queryAndAgentPairs) {
     var KLDivs = mapIndexed(function(queryId, queryAndAgent) {
        var queryPath = makeQueryPath(experimentName, runId, queryId);
        // Hack to extract the facts from each world (since the domain is the same)
        var query = map(function(q) { q.facts || q }, queryAndAgent.query);
        var trueResult = apply(groundTruthModel[queryAndAgent.agent], queryAndAgent.query);
        var learnedResult = apply(learnedModel[queryAndAgent.agent], queryAndAgent.query);
        var KLDivergence = KL(trueResult, learnedResult);

        if (!fs.existsSync(queryPath.dir)) {
            webpplFs.mkdirp(queryPath.dir);
        }
        
        return KLDivergence;
    }, queryAndAgentPairs);

    var expectedKL = listMean(KLDivs)
    webpplFs.write(makeKLStatPath(experimentName, runId).prior, expectedKL);
    return expectedKL;
}

var serializeWorld = function(w) {
    return "doctor(" + (w.facts.doctor.join(',') || "none") + "), " +
        "teacher(" +  (w.facts.teacher.join(',') || "none") + ")";
}

var computeAndWriteResults = function(experimentName, runId, groundTruthModel, learnedModel, queryAndAgentPairs) {
    var queriesAndResults = mapIndexed(function(queryId, queryAndAgent) {
        var queryPath = makeQueryPath(experimentName, runId, queryId);
        // Hack to extract the facts from each world (since the domain is the same)
        var query = map(function(q) { q.facts || q }, queryAndAgent.query);
        var trueResult = apply(groundTruthModel[queryAndAgent.agent], queryAndAgent.query);
        var learnedResult = apply(learnedModel[queryAndAgent.agent], queryAndAgent.query);
        var KLDivergence = KL(trueResult, learnedResult);


        // Write query and result before returning them
        if (!fs.existsSync(queryPath.dir)) {
            webpplFs.mkdirp(queryPath.dir);
        }
        json.write(queryPath.queryPath, query);
        // Process the worlds in the result
        if (queryAndAgent.agent === "L1" || queryAndAgent.agent === "L0") {
            var trueResultNew = marginalize(trueResult, serializeWorld);
            var learnedResultNew = marginalize(learnedResult, serializeWorld);
            json.write(queryPath.resultPath, { trueResultNew, learnedResultNew, KLDivergence });

        }
        else {
            json.write(queryPath.resultPath, { trueResult, learnedResult, KLDivergence });
        }
        
        return {
            query,
            trueResult,
            learnedResult,
            KLDivergence
        }
    }, queryAndAgentPairs);

    var expectedKL = listMean(map(function(x) { x.KLDivergence }, queriesAndResults))
    webpplFs.write(makeKLStatPath(experimentName, runId).posterior, expectedKL);

    return queriesAndResults;
}

var groundTruthFns = {
    and(args) { return args[0] * args[1] },
    or(args) { return 1 - (1 - args[0]) * (1 - args[1]) },
    not(args) { return 1 - args[0] }
}

var testNetworks = function(experimentName, runId, networks, tests) {
    mapObject(function(networkName, test) {
        var networkTestPath = makeNetworkTestPath(experimentName, runId, networkName);
        var networkCorrectnessSummaryPath = makeNetworkCorrectnessSummary(experimentName, runId, networkName);
        var network = networks[networkName];
        var range = _.range.apply(_, test.range);
        var results = Enumerate(function() {
            var args = repeat(test.arity, function() { uniformDraw(range); });
            return snoc(args, network.call(null, Vector(args)));
        }).support();

        webpplCsv.writeCSV(results, networkTestPath);

        // compute distance
        var distance = Math.sqrt(sum(map(function(r) {
            var learned = r[r.length - 1];
            var real = apply(groundTruthFns[networkName], [r]);
            Math.pow(learned - real, 2)
        }, results)))

        webpplFs.write(networkCorrectnessSummaryPath, distance);

    }, tests);
}