@Test
public void egoMinusEgo() {
    try (Transaction tx = db.beginTx()) {

        Demon demon = new Demon(db);
        Node root = nodes.get(0);

        demon.executeEgoMinusEgo(root);

        ResourceIterator<Node> demonNodes = demon.getDemonNodes();
        Iterator<Relationship> demonRelationships = demon.getDemonRelationships();

        int neighCount = 0;
        while (demonNodes.hasNext()) {
            demonNodes.next();
            neighCount++;
        }
        Assert.assertEquals(5, neighCount);

        int relCount = IteratorUtil.asList(demonRelationships).size();
        Assert.assertEquals(2, relCount);

        demon.clearEgoNetwork(root);
    }
}
 

@Override
public int getNodeCount()
{
    int nodeCount = 0;
    try (final Transaction tx = beginUnforcedTransaction())
    {
        try
        {
            nodeCount = IteratorUtil.count(GlobalGraphOperations.at(neo4jGraph).getAllNodes());
            tx.success();
        }
        catch (Exception e)
        {
            tx.failure();
            throw new BenchmarkingException("unable to get node count", e);
        }
    }

    return nodeCount;
}
 

@Override
public double getGraphWeightSum()
{
    int edgeCount = 0;

    try (final Transaction tx = beginUnforcedTransaction())
    {
        try
        {
            edgeCount = IteratorUtil.count(GlobalGraphOperations.at(neo4jGraph).getAllRelationships());
            tx.success();
        }
        catch (Exception e)
        {
            tx.failure();
            throw new BenchmarkingException("unable to get graph weight sum", e);
        }
    }

    return (double) edgeCount;
}
 

private static String importInfo() {
    GraphDatabaseService db = new GraphDatabaseFactory().newEmbeddedDatabase(STORE_DIR);
    try (Transaction tx = db.beginTx()) {
        int nodes = IteratorUtil.count(db.getAllNodes());
        int rels = IteratorUtil.count(GlobalGraphOperations.at(db).getAllRelationships());
        return "Imported nodes " + nodes + " rels " + rels;
    } finally {
        db.shutdown();
    }
}
 

private static String importInfo() {
    GraphDatabaseService db = new GraphDatabaseFactory().newEmbeddedDatabase(STORE_DIR);

    try (Transaction tx = db.beginTx()) {
        int nodes = IteratorUtil.count(db.getAllNodes());
        int rels = IteratorUtil.count(GlobalGraphOperations.at(db).getAllRelationships());
        return "Imported nodes " + nodes + " rels " + rels;
    } finally {
        db.shutdown();
    }
}
 

private static String importInfo() {
    GraphDatabaseService db = new GraphDatabaseFactory().newEmbeddedDatabase(STORE_DIR);

    try (Transaction tx = db.beginTx()) {
        int nodes = IteratorUtil.count(db.getAllNodes());
        int rels = IteratorUtil.count(GlobalGraphOperations.at(db).getAllRelationships());
        return "Imported nodes " + nodes + " rels " + rels;
    } finally {
        db.shutdown();
    }
}
 

private static String assertImport() {
    GraphDatabaseService db = new GraphDatabaseFactory().newEmbeddedDatabase(STORE_DIR);

    try (Transaction tx = db.beginTx()) {
        int nodes = IteratorUtil.count(db.getAllNodes());
        Assert.assertEquals(USERS, nodes);
        int rels = IteratorUtil.count(GlobalGraphOperations.at(db).getAllRelationships());
        Assert.assertEquals(FRIENDSHIPS, rels);
        return "Imported nodes " + nodes + " rels " + rels;
    } finally {
        db.shutdown();
    }
}
 

public void writeOutContent(String filename) {
	GlobalGraphOperations ops = GlobalGraphOperations.at(graphDb);
	ExecutionEngine engine = new ExecutionEngine(graphDb);

	try (FileWriter writer = new FileWriter(filename);
			Transaction tx = graphDb.beginTx()) {
		for (Node n : ops.getAllNodes()) {
			writer.write("[" + n.getId() + "," + n.getProperty(PROP_NAME)
					+ ",[");
			Iterator<Node> connected = engine.execute(
					"START s=node(" + n.getId()
							+ ") MATCH s-[r]->n RETURN n").columnAs("n");
			for (Node e : IteratorUtil.asIterable(connected)) {
				Iterator<String> rel = engine.execute(
						"START s=node(" + n.getId() + "), e=node("
								+ e.getId()
								+ ") MATCH s-[r]->e RETURN r.value")
						.columnAs("r.value");
				String relVal = rel.hasNext()? rel.next() : "";
				writer.write("[" + e.getId() + ","
						+ relVal + "],");
			}
			writer.write("]]\n");
		}
		tx.success();
	} catch (IOException e1) {
		e1.printStackTrace();
	}
}
 

private static void getNodeHashMap(Long id, GraphDatabaseService gdb, Cache<Long, HashMap<String, Object>> cache) {
    Node thisNode = gdb.getNodeById(id);
    List<String> keys = new ArrayList<>();
    HashMap<String, Object> nodeMap = new HashMap<>();
    IteratorUtil.addToCollection(thisNode.getPropertyKeys(), keys)
            .stream()
            .forEach(n -> nodeMap.put(n, thisNode.getProperty(n)));
    nodeMap.put("id", id);
    cache.put(id, nodeMap);
}
 

public static Map<Long, Integer> getTermFrequencyMapForDocument(GraphDatabaseService db, Long classId)
{
    Map<Long, Integer> termDocumentMatrix;

    String cacheKey = "TERM_DOCUMENT_FREQUENCY_" + classId;

    if(vectorSpaceModelCache.getIfPresent(cacheKey) == null) {
        Node classNode = db.getNodeById(classId);

        termDocumentMatrix = new HashMap<>();

        IteratorUtil.asCollection(db.traversalDescription()
                .depthFirst()
                .relationships(withName("HAS_CLASS"), Direction.INCOMING)
                .evaluator(Evaluators.fromDepth(1))
                .evaluator(Evaluators.toDepth(1))
                .traverse(classNode)).stream()
                .forEach(p ->
                {
                    int matches = (Integer) p.lastRelationship().getProperty("matches");
                    termDocumentMatrix.put(p.endNode().getId(), matches);
                });



        vectorSpaceModelCache.put(cacheKey, termDocumentMatrix);
    }
    else
    {
        termDocumentMatrix =  (Map<Long, Integer>)vectorSpaceModelCache.getIfPresent(cacheKey);
    }

    return termDocumentMatrix;
}
 

public static int getDocumentSize(GraphDatabaseService db)
{
    int documentSize;
    String cacheKey = "GLOBAL_DOCUMENT_SIZE";
    if(vectorSpaceModelCache.getIfPresent(cacheKey) == null) {
        documentSize = IteratorUtil.count(GlobalGraphOperations.at(db).getAllNodesWithLabel(DynamicLabel.label("Class")));
        vectorSpaceModelCache.put(cacheKey, documentSize);
    }
    else
    {
        documentSize = (Integer)vectorSpaceModelCache.getIfPresent(cacheKey);
    }
    return documentSize;
}
 

public static double getFeatureMatchDistribution(GraphDatabaseService db, Long patternId)
{
    Transaction tx = db.beginTx();
    Node startNode = db.getNodeById(patternId);

    // Feature match distribution
    List<Double> matches = IteratorUtil.asCollection(db.traversalDescription()
            .depthFirst()
            .relationships(withName("HAS_CLASS"), Direction.OUTGOING)
            .evaluator(Evaluators.fromDepth(1))
            .evaluator(Evaluators.toDepth(1))
            .traverse(startNode)
            .relationships())
            .stream()
            .map(p -> ((Integer)p.getProperty("matches")).doubleValue())
            .collect(Collectors.toList());

    tx.success();
    tx.close();

    double variance = 1.0;

    if(matches.size() > 1) {
        Double[] matchArr = matches.toArray(new Double[matches.size()]);
        // Get the standard deviation
        DescriptiveStatistics ds = new DescriptiveStatistics();
        matches.forEach(m -> ds.addValue(m.doubleValue() / StatUtils.sum(ArrayUtils.toPrimitive(matchArr))));
        variance = ds.getStandardDeviation();
    }

    return variance;
}
 

public static int getDocumentSizeForFeature(GraphDatabaseService db, Long id)
{
    int documentSize;

    String cacheKey = "DOCUMENT_SIZE_FEATURE_" + id;

    if(vectorSpaceModelCache.getIfPresent(cacheKey) == null) {
        Node startNode = db.getNodeById(id);

        Iterator<Node> classes = db.traversalDescription()
                .depthFirst()
                .relationships(withName("HAS_CLASS"), Direction.OUTGOING)
                .evaluator(Evaluators.fromDepth(1))
                .evaluator(Evaluators.toDepth(1))
                .traverse(startNode)
                .nodes().iterator();

        documentSize = IteratorUtil.count(classes);

        vectorSpaceModelCache.put(cacheKey, documentSize);
    }
    else
    {
        documentSize = (Integer)vectorSpaceModelCache.getIfPresent(cacheKey);
    }

    return documentSize;
}
 

public static void main(String[] args) {
       String zipFile = "data/cineasts_12k_movies_50k_actors_2.1.6.zip";
       String path = "data/cineasts_12k_movies_50k_actors.db/";

       try {
           FileUtils.deleteRecursively(new File(path));
           extractFolder(zipFile);
       } catch (Exception e) {
           e.printStackTrace();
           System.exit(1);
       }

       long nodeCount, relsCount;
	
	// Open a database instance
       GraphDatabaseService g = new GraphDatabaseFactory()
               .newEmbeddedDatabaseBuilder(path)
               .setConfig(GraphDatabaseSettings.allow_store_upgrade, "true")
               .newGraphDatabase();
       try (Transaction tx = g.beginTx() ) {
		nodeCount = IteratorUtil.count( GlobalGraphOperations.at(g).getAllNodes() );
		relsCount = IteratorUtil.count( GlobalGraphOperations.at(g).getAllRelationships() );
		tx.success();
	}
	
	System.out.println("Node count: "+nodeCount);
       System.out.println("Rel count: " + relsCount);

       // Declare the GraphAlgoEngine on the database instance
	GraphAlgoEngine engine = new GraphAlgoEngine(g);
	if( args.length > 1 && args[1].equals("off") )
		engine.disableLogging();

       Louvain louvain = new Louvain(g);
       louvain.execute();
       LouvainResult result = louvain.getResult();
       for (int layer : result.layers()) {
           System.out.println("Layer " + layer + ": " + result.layer(layer).size() + " nodes");
       }

       LabelPropagation lp = new LabelPropagation();
       // Starts the algorithm on the given graph g
	engine.execute(lp);
	Long2LongMap communityMap = lp.getResult();
	long totCommunities = new LongOpenHashSet( communityMap.values() ).size();
       System.out.println("There are " + totCommunities + " communities according to Label Propagation");

	DirectedModularity modularity = new DirectedModularity(g);
	engine.execute(modularity);
       System.out.println("The directed modularity of this network is " + modularity.getResult());

       UndirectedModularity umodularity = new UndirectedModularity(g);
	engine.execute(umodularity);
       System.out.println("The undirected modularity of this network is " + umodularity.getResult());

       engine.clean(lp); // Now you can clean Label propagation results

       TriangleCount tc = new TriangleCount();
	engine.execute(tc);
	Long2LongMap triangleCount = tc.getResult();
	Optional<Long> totalTriangles = triangleCount.values().stream().reduce( (x, y) -> x + y );
	System.out.println("There are "+totalTriangles.get()+" triangles");

	PageRank pr = new PageRank(g);
	engine.execute(pr);
	Long2DoubleMap ranks = pr.getResult();
	engine.clean(pr);
	Optional<Double> res = ranks.values().parallelStream().reduce( (x, y) -> x + y );
	System.out.println("Check PageRank sum is 1.0: "+ res.get());

	ConnectedComponents cc = new ConnectedComponents();
	engine.execute(cc);
	Long2LongMap components = cc.getResult();
	engine.clean(cc);
	int totalComponents = new LongOpenHashSet( components.values() ).size();
	System.out.println("There are "+ totalComponents+ " different connected components");
	
	StronglyConnectedComponents scc = new StronglyConnectedComponents();
	engine.execute(scc);
	components = scc.getResult();
	engine.clean(scc);
	totalComponents = new LongOpenHashSet( components.values() ).size();
	System.out.println("There are "+ totalComponents+ " different strongly connected components");
	
	// Don't forget to shutdown the database
	g.shutdown();
}
 

public static Path exportSubgraphToHDFS(GraphDatabaseService db) throws IOException, URISyntaxException {
    FileSystem fs = FileUtil.getHadoopFileSystem();
    Path pt = new Path(ConfigurationLoader.getInstance().getHadoopHdfsUri() + EDGE_LIST_RELATIVE_FILE_PATH.replace("/{job_id}", ""));
    BufferedWriter br = new BufferedWriter(new OutputStreamWriter(fs.create(pt)));

    Transaction tx = db.beginTx();

    // Get all nodes in the graph
    Iterable<Node> nodes = GlobalGraphOperations.at(db)
            .getAllNodes();

    br.write("# Adacency list" + "\n");

    int nodeTotal = IteratorUtil.count(nodes);
    final int[] nodeCount = {0};
    final int[] pathCount = {0};
    int pathCountBlocks = 10000;

    int size = IteratorUtil.count(nodes.iterator());

    //System.out.println(nodes.spliterator().trySplit().estimateSize());

    // Fork join

    nodes.iterator().forEachRemaining(n -> {
        // Filter nodes by all paths connected by the relationship type described in the configuration properties
        Iterable<org.neo4j.graphdb.Path> nPaths = db.traversalDescription()
                .depthFirst()
                .relationships(withName(ConfigurationLoader.getInstance().getMazerunnerRelationshipType()), Direction.OUTGOING)
                .evaluator(Evaluators.fromDepth(1))
                .evaluator(Evaluators.toDepth(1))
                .traverse(n);

        for (org.neo4j.graphdb.Path path : nPaths) {
            try {
                String line = path.startNode().getId() + " " + path.endNode().getId();
                br.write(line + "\n");
                pathCount[0]++;
                if (pathCount[0] > pathCountBlocks) {
                    pathCount[0] = 0;
                    System.out.println("Mazerunner Export Status: " + MessageFormat.format("{0,number,#%}", ((double) nodeCount[0] / (double) nodeTotal)));
                }
            } catch (Exception ex) {
                System.out.println(ex.getMessage());
            }
        }
        nodeCount[0]++;
    });

    System.out.println("Mazerunner Export Status: " + MessageFormat.format("{0,number,#.##%}", 1.0));

    br.flush();
    br.close();
    tx.success();
    tx.close();

    return pt;
}
 

/**
 * Experimental: Create affinity relationships between patterns that are encountered together during training.
 * @param graphManager is the global graph manager for managing an optimized cache of graph data
 * @param db is the Neo4j database service
 * @param tree is the decision tree for pattern matching
 * @param patternMatchers is the set of pattern matchers produced from the decision tree
 */
private static void createAffinityRelationships(GraphManager graphManager, GraphDatabaseService db, DecisionTree<Long> tree, Map<Long, Integer> patternMatchers) {
    List<String> toFrom = new ArrayList<>();
    for (Long startId : patternMatchers.keySet()) {
        // Get or create the affinity relationship
        try(Transaction tx = db.beginTx()) {
            patternMatchers.keySet().stream().filter(endId -> startId != endId).forEach(endId -> {
                Node startNode = db.getNodeById(startId > endId ? endId : startId);
                Node endNode = db.getNodeById(startId > endId ? startId : endId);
                String key = startNode.getId() + "_" + endNode.getId();

                // If startNode matches the end node
                Pattern generalMatcher = Pattern.compile((String) startNode.getProperty("pattern"));
                Matcher regexMatcher = generalMatcher.matcher(((String) endNode.getProperty("phrase")).replaceAll("(\\{[01]\\})", "word"));

                if (!toFrom.contains(key) && !regexMatcher.find()) {
                    // Depth of the start node must be > 2
                    PathFinder<Path> depthFinder = GraphAlgoFactory.shortestPath(PathExpanders.forTypeAndDirection(withName("NEXT"), Direction.OUTGOING), 100);
                    int depth1 = IteratorUtil.count(depthFinder.findSinglePath(db.getNodeById(tree.root()), startNode).nodes().iterator());
                    int depth2 = IteratorUtil.count(depthFinder.findSinglePath(db.getNodeById(tree.root()), endNode).nodes().iterator());

                    if (depth1 > 3 && depth2 > 3 && depth2 > depth1) {
                        // Eliminate descendant
                        PathFinder<Path> finder = GraphAlgoFactory.shortestPath(PathExpanders.forTypeAndDirection(withName("NEXT"), Direction.OUTGOING), 100);
                        Path findPath = finder.findSinglePath(startNode, endNode);

                        if (findPath == null) {
                            toFrom.add(key);
                            // Get or create the affinity relationship
                            affinityRelationshipCache.getOrCreateRelationship(startNode.getId(), endNode.getId(), db, graphManager, true);
                        }
                    }
                }
            });
            tx.success();
            tx.close();
        } catch (Exception ex) {
            throw ex;
        }
    }
}
 

public double getNodeInDegree(Node node)
{
    Iterable<Relationship> rel = node.getRelationships(Direction.OUTGOING, RelTypes.SIMILAR);
    return (double) (IteratorUtil.count(rel));
}
 

public double getNodeOutDegree(Node node)
{
    Iterable<Relationship> rel = node.getRelationships(Direction.INCOMING, RelTypes.SIMILAR);
    return (double) (IteratorUtil.count(rel));
}