Not to be confused with nogegraph.pl for the E2 Nodegel Visualiser, this code is to begin mapping the nodegel, in the graph theoretical sense. Please /msg me with comments or criticism, and save your downvotes for my edev: mapping the nodegel write-up. This is posted here because it's rather long, and only of interest to a minority of noders...
I've been using a Windows 98 machine with Mingw32, but it should work fine on Linux too. Use something like "g++ -g nodes2graphviz.cc -o nodes2graphviz.exe -Wall -pedantic -Wno-unknown-pragmas -ltxml -lcurl -lwsock32 -lws2_32" to compile it for windows, having compiled tinyxml and libcurl as libraries first, and put the necessary files in to your lib/ and include/ directories.
Current usage instructions go something like:
Usage: nodes2graphviz [Options]
Options:
-username <name> Where <name> is your everything2 username.
-password <pass> Where <pass> is your e2 password, and
if no password is given, user_search.xml
must already exist in directory username/xml/
-type <type> where <type> is either neato or twopi
/////////////////////////////////////////////////////////////////////////////
// //
// nodes2graphviz.cpp //
// //
// Code provided "as is". I'm pretty sure all the libraries //
// used are Free (as in speech). This is too, and though I'm //
// not too picky about licensing, I suppose I should choose one. //
// //
// OK, distributed under the Artistic License. That'll do :) //
// //
/////////////////////////////////////////////////////////////////////////////
// //
// Version 1.0, May 10th 2002. //
// //
// Takes a username and password, connects to everything2, //
// saves the output of User Search XML Ticker, parses the //
// file and fetches all of the user's nodes, parses the nodes //
// and produces a .dot file containing a graph of all the //
// nodes and their softlink destinations. //
// //
// Requires TinyXml(1), libCurl(2) and AT&T's GraphViz(3). //
// //
// (1) http://www.grinninglizard.com/tinyxml/ //
// or //
// http://www.sourceforge.net/projects/tinyxml/ //
// //
// (2) http://curl.haxx.se/ //
// or //
// http://curl.sourceforge.net/ //
// //
// (3) http://www.research.att.com/sw/tools/graphviz/ //
// or //
// http://www.graphviz.org/ //
// //
// Since not everyone speaks c++, I'm willing to offer a ready //
// compiled version or help with compiling to anyone who asks. //
// //
/////////////////////////////////////////////////////////////////////////////
// //
// Version 1.0.1, May 12th 2002, Just_Tom //
// Tidied up code, no warnings! //
// Tweaked the graph settings //
// Fixed bug where selecting (A)ll would miss downloading first node //
// Added ranksep for twopi graphs //
// //
// Version 1.1, May 12th 2002, Just_Tom //
// Added command line arguments //
// //
// Version 1.2, May 13th 2002, Just_Tom //
// Added different settings for neato and twopi, looks reasonable for //
// my 72 nodes... //
// //
// Version 1.2.1, May 13th 2002, Just_Tom //
// Added &no_doctext=1 request for fetching nods, since writeup text //
// is not needed (thanks insanefuzzie) //
// //
/////////////////////////////////////////////////////////////////////////////
// C stuff
#include <cstdio>
#include <cstdlib>
// C++ stuff
#include <string>
#include <vector>
#include <fstream>
#include <sstream>
// HTTP library, (libcurl/cURL)
#include <curl/curl.h>
#include <curl/types.h>
#include <curl/easy.h>
// XML libary, TinyXml
#include <tinyxml.h>
using namespace std;
// forward declare for typedefs
class NodeParser;
class UserSearchParser;
// mostly shorter type names
typedef vector<string> Strings;
typedef vector<string>::iterator StringsIterator;
typedef vector<NodeParser*> pNodeParsers;
typedef vector<NodeParser*>::iterator pNodeParserIterator;
//
// Parser classes to wrap around TinyXml...
//
class NodeParser {
private:
TiXmlDocument* document;
public:
NodeParser( const string& aFilename ):
document( new TiXmlDocument( aFilename ) )
{
if ( !document->LoadFile() )
{
cerr << "TiXml Error='"
<< document->ErrorDesc().c_str()
<< "'. Exiting." << endl;
exit( 1 );
} // if
}
~NodeParser() { delete document; }
// edgeSep for Graphviz will normally be "--" or "->"
Strings getEdges(const string& edgeSep) const
{
Strings edges; // somewhere to put edges
// navigate to root node
TiXmlElement* node = document->FirstChildElement( "node" );
string thisNodeId = *(node->Attribute( "node_id" ));
// get the softlinks
TiXmlElement* softlinks = node->FirstChildElement( "softlinks" );
// fetch all nodeIDs
for ( TiXmlElement* e2link = softlinks->FirstChildElement( "e2link" );
e2link;
e2link = e2link->NextSiblingElement( "e2link" ) )
{
string linkTargetId( *(e2link->Attribute( "node_id" )) );
edges.push_back( thisNodeId + edgeSep + linkTargetId );
} // for
return edges;
} // getEdges
}; // NodeParser
class UserSearchParser {
private:
TiXmlDocument* document;
public:
UserSearchParser( const string& aFilename ):
document( new TiXmlDocument( aFilename ) )
{
if ( !document->LoadFile() ) {
cerr << "TiXml Error='" << document->ErrorDesc().c_str()
<< "'. Exiting." << endl;
exit( 1 );
}
}
~UserSearchParser() { delete document; }
vector<string> getNodeIDs() const
{
// navigate to root node
TiXmlNode* userSearch = document->FirstChild( "USERSEARCH" );
// somewhere to put "vertices"
Strings nodeIDs;
// fetch all nodeIDs
for ( TiXmlElement* writeup = userSearch->FirstChildElement( "writeup" );
writeup;
writeup = writeup->NextSiblingElement( "writeup" ) )
{
nodeIDs.push_back( *(writeup->Attribute("parent_e2node")) );
}
return nodeIDs;
} // getNodeIDs
}; // UserSearchParser
//
// function prototypes
//
template<class T> string toString(const T&);
size_t write_data( void*, size_t, size_t, void* );
bool url2File( const string&, const string& );
int main( int argc, char* argv[] )
{
cout << endl;
//
// Process arguments for username and password...
//
//
if ( argc != 3 && argc != 5 && argc != 7)
{
cout << "ERROR: not enough arguments..." << endl << endl
<< "Usage: nodes2graphviz [Options]" << endl
<< "Options: " << endl
<< "\t-username <name> Where <name> is your everything2 username." << endl
<< "\t-password <pass> Where <pass> is your e2 password, and" << endl
<< "\t if no password is given, user_search.xml" << endl
<< "\t must already exist in directory username/xml/" << endl
<< "\t-type <type> where <type> is either neato or twopi" << endl;
exit(1);
}
// default settings...
bool gotPass = false;
bool gotUser = false;
bool twopi = false;
bool neato = true;
string username;
string password;
Strings args(argc);
for (int i = 1; i < argc; i++)
{
args.push_back(string(argv[i]));
}
cout << "Parsing options..." << endl;
for (StringsIterator i = args.begin(); i != args.end(); i++)
{
if (*i == string("-username"))
{
username = *(++i);
gotUser = true;
cout << "\tUsername set to " << username << "..." << endl;
}
else if (*i == string("-password"))
{
password = *(++i);
gotPass = true;
cout << "\tPassword set to " << password << "..." << endl;
}
else if (*i == string("-type") && *(i+1) == string("twopi"))
{
twopi = true;
neato = false;
cout << "\tMaking a graph for twopi..." << endl;
}
}
if (!gotUser)
{
cout << "\tNo username supplied... exiting" << endl;
exit(1);
}
if (!gotPass)
{
cout << "\tNo password supplied..." << endl;
}
if (neato)
{
cout << "\tMaking a graph for neato..." << endl;
}
//
// Declare a load of useful strings...
//
//
// files and paths
const string xmlExt(".xml");
// if there's just you, use these:
//const string userpath("");
//const string xmlpath("");
//const string userSearchFileName("user_search.xml");
//const string dotfile("/e2graph.dot");
// use these if you're running it for more than one person.
// You may have to make the directories first :(
const string userpath = username;
const string xmlpath = userpath + string("/xml/");
const string userSearchFileName = xmlpath + string("user_search.xml");
const string dotfile = userpath + string( "/e2graph.dot" );
// URLs
const string e2URL("http://www.everything2.com");
const string userSearchURL = e2URL
+ string("/?node=User%20Search%20XML%20Ticker&op=login&user=") +
username + string("&passwd=") + password;
const string e2NodeAddress = e2URL + string("/index.pl?node_id=" );
const string homenodeURL = string("?node=")
+ username + string("&type=user");
const string e2nodeURL = e2URL + string("?node_id="); // + node_id
const string e2XmlRequest( "&displaytype=xmltrue" );
const string e2NoDocRequest("&no_doctext=1");
char temp;
bool done = false;
//
// Download info from [User Search XML Ticker]...
// TODO: use [User Search XML Ticker II]
//
if (gotPass) {
cout << endl << "Connect to e2 to download most recent user search? (Y)es/(N)o: ";
while (!done)
{
cin >> temp;
if (temp == 'Y' || temp == 'y')
{
done = url2File( userSearchURL, userSearchFileName );
}
else if (temp == 'N' || temp == 'n')
{
done = true;
}
else
{
cout << "Bad input, try again? (Y)es/(N)o: ";
}
}
}
else
{
cout << endl << "No password supplied so skipping download of user search... " << endl;
}
//
// Get node_ids from saved user_search.xml...
//
//
cout << endl << "Attempting to open and parse "
<< userSearchFileName << " to get node_ids..." << endl;
UserSearchParser parser( userSearchFileName );
Strings nodeIDs = parser.getNodeIDs();
//
// Download all nodes...
//
//
cout << "About to annoy E2 by downloading ALL your nodes..." << endl;
int count = 1;
bool all = false;
bool none = false;
for ( StringsIterator i = nodeIDs.begin(); i != nodeIDs.end() && !none; i++ )
{
if (!all)
{
cout << "Get node_id " << *i << " ("
<< count << " of " << nodeIDs.size()
<< ")? (Y)es, (A)ll, (N)one: ";
cin >> temp;
if (temp == 'Y' || temp == 'y')
{
url2File( e2NodeAddress + *i + e2XmlRequest + e2NoDocRequest, xmlpath + *i + xmlExt );
}
else if (temp == 'N' || temp == 'n')
{
cout << "Aborting all transfers..." << endl;
none = true;
}
else if (temp == 'A' || temp == 'a')
{
url2File( e2NodeAddress + *i + e2XmlRequest, xmlpath + *i + xmlExt );
cout << "Fetching all..." << endl;
all = true;
}
}
else
{
cout << "Geting node_id " << *i << " ("
<< count << " of " << nodeIDs.size() << ")" << endl;
url2File( e2NodeAddress + *i + e2XmlRequest, *i + xmlExt );
}
count++;
} // for
//
// Graphviz settings...
// TODO load these from a file
//
int maxNumNodes = 0;
if (neato) {
maxNumNodes = 100;
cout << "Limiting to 100 nodes for neato sanity... " << endl;
}
else if (twopi) {
maxNumNodes = 5000;
cout << "Limiting to 500 nodes for twopi sanity... " << endl;
}
int maxNumSoftLinks = 20; // (can't get more without logging in anyhow)
bool joinOwnerNodes = false; // these sometimes help
bool spaceNodes = false; // make it look nicer
bool weightLinks = false; // but sometimes don't :(
bool ownerIsNode = twopi; // include a node for the owner,
// and it to all their nodes
bool diEdges = !neato; // directed edges have arrows and can be
// drawn with "dot"
//
// GraphViz strings...
// You'll probably want to edit these.
//
string ownerNodeSettings, nodeSettings, nameNodeSettings, edgeSettings, graphSettings, ownerToNodesSettings;
if (neato)
{
ownerNodeSettings = string(
"[shape=circle, style=filled, color=red, label=\"\", width=10, height=10]"
);
nodeSettings = string(
"[shape=circle, style=filled, color=skyblue, label=\"\", width=5, height=5]"
);
edgeSettings = string("[style=bold, len=25]");
graphSettings = string("size=\"50,50\""); // otherwise get massive images
}
else // if (twopi)
{
ownerNodeSettings = string(
"[shape=circle, style=filled, color=red, label=\"\", width=1.0, height=1.0]"
);
nodeSettings = string(
"[shape=circle, style=filled, color=skyblue, label=\"\", width=0.5, height=0.5]"
);
nameNodeSettings = string ("[URL=\"") +
homenodeURL + string("\",") +
string(
"shape=circle, style=filled, color=red, width=8, height=8, fontsize=160]"
);
edgeSettings = string("[style=dashed]");
graphSettings = string("size=\"50,50\""); // otherwise get massive images
ownerToNodesSettings = string("[style=solid]");
}
//
// Output all relevant info to a GraphViz .dot file
//
//
// use a parser for every node
pNodeParsers parsers;
// open dotfile as an ostream
ofstream dotstream;
dotstream.open( dotfile.c_str() );
// check if worked
if ( dotstream.fail() )
{
cerr << "Could not save to file \"" << dotfile << '"' << endl;
exit( 1 );
} // if
// open a graph in dotfile
if (diEdges)
dotstream << "digraph G {" << endl;
else
dotstream << "graph G {" << endl;
// set global graph settings
dotstream << '\t' << graphSettings << endl;
if (spaceNodes)
dotstream << "\toverlap=scale" << endl;
if (ownerIsNode)
{
dotstream << "\tcenter=" << username << endl
<< "\tranksep=30" << endl;
}
// open the owner nodes
dotstream << "\tsubgraph N { " << endl
<< "\t\tURL=\"" << e2URL << "\"" << endl;
if (ownerIsNode)
{
cout << "Making " << username << " their own node..."<< endl;
dotstream << "\t\t" << username << ' ' << nameNodeSettings << endl;
}
// set drawing parameters for owner's nodes
dotstream << "\t\tnode " << ownerNodeSettings << endl;
int nodeCount = 0; // so we know if we've reached the limit
for (StringsIterator i = nodeIDs.begin();
i != nodeIDs.end() && nodeCount < maxNumNodes; i++ )
{
cout << "Attempting to make a parser for "
<< xmlpath + *i + xmlExt << endl;
// make a parser for each node's xml data
parsers.push_back( new NodeParser( xmlpath + *i + xmlExt ) );
// record all of owner's nodes in dotfile
// so they can be drawn as specified above
dotstream << "\t\t" << *i << " [URL=\""
<< e2nodeURL << *i << "\"]" << endl;
nodeCount++;
} // for
// determine edge type
string edgeString;
if (diEdges)
edgeString = string(" -> ");
else
edgeString = string(" -- ");
if (ownerIsNode)
{
cout << "Joining " << username << "\'s nodes to their node..."<< endl;
dotstream << "\t\tedge " << ownerToNodesSettings << endl;
// go through all the nodes and join them to the owner node
nodeCount = 0;
for (StringsIterator i = nodeIDs.begin();
i != nodeIDs.end() && nodeCount < maxNumNodes; i++ )
{
dotstream << "\t\t" << username << edgeString << *i << endl;
nodeCount++;
}
} // if
if (joinOwnerNodes)
{
cout << "Joining " << username
<< "\'s nodes with invisible edges..."<< endl;
// next make neato join all the owner nodes with long invisible lines
dotstream << endl << "\t\tedge [len=15,style=invis];" << endl;
// go through all the nodes an put them in a big -- list -- like -- this
vector<string>::iterator i = nodeIDs.begin();
dotstream << "\t\t" << *i;
nodeCount = 1;
while ( i != nodeIDs.end() && nodeCount < maxNumNodes ) {
// join all of owner's nodes with invisible edges to aid layout
dotstream << edgeString << *i;
i++;
nodeCount++;
}
} // if
// close the subgraph of possibly joined owner's nodes and ownerNode
dotstream << "\t}" << endl;
// set drawing parameters
dotstream << "\tnode " << nodeSettings << endl;
dotstream << "\tedge " << edgeSettings << endl;
// output actual edges needed
cout << "Writing graph structure to " << dotfile << endl;
Strings edges;
for ( pNodeParserIterator i = parsers.begin(); i != parsers.end(); i++ )
{
// helpful output (we're doing a node)
cout << 'O';
// get the edges from each node
edges = (*i)->getEdges(edgeString);
int count = 0;
// weight the edges?
int linkStrength = maxNumSoftLinks;
for ( StringsIterator j = edges.begin();
j != edges.end() && count < maxNumSoftLinks; j++ )
{
// helpful output (we're doing an edge)
cout << '-';
// output each edge
dotstream << "\t" << *j;
if (weightLinks)
{
// weight the links
dotstream << "[w=" << toString(linkStrength) << "]; ";
linkStrength--;
}
dotstream << endl;
count++;
} // for each edge
} // for each node
// close the graph
dotstream << '}' << endl;
cout << "done." << endl;
// tidy
dotstream.close();
return 0;
} // main
// helpful function, token amount of checking...
template< class T >
string toString(const T& x)
{
ostringstream o;
if (o << x)
return o.str();
else
return string("conversion error");
} // toString
size_t write_data(void *ptr, size_t size, size_t nmemb, void *stream)
{
return fwrite(ptr, size, nmemb, (FILE *)stream);
}
bool url2File(const string& URL, const string& filename)
{
cout << endl << "\tInitialising cURL session to " << URL << "..." << endl;
CURL *curl_handle = curl_easy_init();
curl_easy_setopt(curl_handle, CURLOPT_URL, URL.c_str());
cout << "\tOpening file " << filename << "..." << endl;
FILE *file = fopen(filename.c_str(), "w+");
if (file == NULL)
{
cout << "failed!" << endl;
curl_easy_cleanup(curl_handle);
return false;
}
// TODO check if file exists, prompt for over-write
cout << "\tTelling cURL to write to " << filename << "..." << endl;
curl_easy_setopt(curl_handle, CURLOPT_FILE, file);
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_data);
cout << "\tConnecting to " << URL << "..." << endl;
curl_easy_perform(curl_handle);
cout << "\tWriting " << filename << "..." << endl;
fclose(file);
cout << "\tClosing cURL session..." << endl;
curl_easy_cleanup(curl_handle);
cout << endl;
return true;
} // url2File