changeset 11:701b90894fec

Added the C++/Python check-ins transformer
author Michal Kawalec <mkawalec@lavabit.com>
date Sun, 02 Jun 2013 06:53:16 +0200
parents f3db2ed80648
children df4ae57c7e2d
files boost-py/Makefile boost-py/hello_ext.C boost-py/timeit.py
diffstat 3 files changed, 227 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/boost-py/Makefile	Sun Jun 02 06:53:16 2013 +0200
@@ -0,0 +1,18 @@
+# location of the Python header files
+ 
+PYTHON_VERSION = 2.7
+PYTHON_INCLUDE = /usr/include/python$(PYTHON_VERSION)
+ 
+# location of the Boost Python include files and library
+ 
+BOOST_INC = /usr/include
+BOOST_LIB = /usr/lib
+ 
+# compile mesh classes
+TARGET = hello_ext
+ 
+$(TARGET).so: $(TARGET).o
+	g++ -shared -Wl,--export-dynamic $(TARGET).o -L$(BOOST_LIB) -lboost_python -L/usr/lib/python$(PYTHON_VERSION)/config -lpython$(PYTHON_VERSION) -o $(TARGET).so
+ 
+$(TARGET).o: $(TARGET).C
+	g++ -I$(PYTHON_INCLUDE) -I$(BOOST_INC) -fPIC -c $(TARGET).C
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/boost-py/hello_ext.C	Sun Jun 02 06:53:16 2013 +0200
@@ -0,0 +1,191 @@
+#include <string>
+#include <vector>
+#include <iostream>
+#include <set>
+#include <boost/python.hpp>
+#include <boost/python/suite/indexing/vector_indexing_suite.hpp>
+using namespace boost::python;
+
+typedef std::vector<int> intvec;
+typedef std::pair<int, std::vector<int> > pairlist;
+#define FOREACH BOOST_FOREACH
+
+struct Checkin
+{
+    std::vector<int> ids;
+};
+
+struct Checkins
+{
+    Checkins(list ids): checkins(convert(ids)) {};
+    std::vector<std::vector<int> > checkins;
+
+    list compute(const int master)
+    {
+        // Get all the distinct ids in the checkins
+        std::set<int> hash;
+        for(int i=0; i < checkins.size(); ++i)
+        {
+            for(int j=0; j<checkins[i].size(); ++j)
+            {
+                hash.insert(checkins[i][j]);
+            }
+        }
+
+        std::vector<pairlist> score_list(hash.size());
+        int index = 0;
+
+        for(std::set<int>::iterator it=hash.begin(); it != hash.end(); ++it)
+        {
+            int score = 0;
+            score_list[index].first = *it;
+            score_list[index].second.reserve(checkins.size());
+
+            for(int i=0; i < checkins.size(); ++i)
+            {
+                for(int j=0; j < checkins[i].size(); ++j)
+                {
+                    if(checkins[i][j] == *it)
+                    {
+                        score += 2;
+                        break;
+                    }
+                }
+                score -= 1;
+                score_list[index].second.push_back(score);
+            }
+            score_list[index].second = derivative(score_list[index].second);
+            index += 1;
+        }
+        return create_checkins(remove_nonmatching(score_list));
+    } 
+
+    list create_checkins(std::vector<pairlist> v)
+    {
+        std::vector<Checkin> ret(v[0].second.size());
+        for(int i=0; i < v.size(); ++i)
+        {
+            v[i].second.resize(v[i].second.size()+3);
+        }
+
+        for(int i=0; i < v[0].second.size()-3; ++i)
+        {
+            Checkin tmp;
+            tmp.ids.reserve(7);
+
+            for(int j=0; j < v.size(); ++j)
+            {
+                if(v[j].second[i] > 0 && 
+                        (v[j].second[i+1] > 0 ||
+                         v[j].second[i+2] > 0))
+                    tmp.ids.push_back(v[j].first);
+                else if(i > 1) {
+                    if(v[j].second[i] > 0 &&
+                        (v[j].second[i-1] > 0 ||
+                         v[j].second[i-2] > 0)) {
+                        tmp.ids.push_back(v[j].first);
+                    }
+                }
+            }
+            ret[i] = tmp;
+        }
+        return to_python(ret);
+    }
+                
+
+    std::vector<pairlist> remove_nonmatching(std::vector<pairlist>& v)
+    {
+        std::vector<pairlist> ret;
+
+        FOREACH(pairlist& pl, v)
+        {
+            if(is_matching(pl))
+                ret.push_back(pl);
+        }
+        return ret;
+    }
+
+    bool is_matching(pairlist& pl)
+    {
+        if(pl.second.size() < 3)
+            return false;
+
+        for(int i=0; i < pl.second.size()-3; ++i)
+        {
+            if(pl.second[i] > 0 &&
+               pl.second[i+1] > 0 &&
+               pl.second[i+2] > 0)
+                return true;
+        }
+        
+        return false;
+    }
+
+    intvec derivative(std::vector<int>& points)
+    {
+        intvec ret;
+        ret.reserve(points.size()-1);
+
+        for(int i=0; i < points.size()-1; ++i)
+        {
+            ret.push_back(points[i+1]-points[i]);
+        }
+        return ret;
+    }
+
+
+    private:
+        list to_python(std::vector<pairlist> v)
+        {
+            list ret;
+            FOREACH(pairlist& pl, v)
+            {
+                list temp;
+                FOREACH(int& tmp, pl.second)
+                {
+                    temp.append(tmp);
+                }
+                ret.append(make_tuple(pl.first, temp));
+            }
+            return ret;
+        }
+
+        list to_python(std::vector<Checkin>& v)
+        {
+            list ret;
+            FOREACH(Checkin& ch, v)
+            {
+                list tmp;
+                FOREACH(int& id, ch.ids)
+                {
+                    tmp.append(id);
+                }
+                ret.append(tmp);
+            }
+            return ret;
+        }
+
+        std::vector<std::vector<int> > convert(list& list)
+        {
+            std::vector<std::vector<int> > ret(len(list));
+
+            for(int i=0; i < len(list); ++i)
+            {
+                ret[i].resize(len(list[i]));
+                for(int j=0; j < len(list[i]); ++j)
+                {
+                    ret[i][j] = extract<int>(list[i][j]);
+                }
+            }
+            return ret;
+        }
+};
+
+
+
+BOOST_PYTHON_MODULE(hello_ext)
+{
+    class_<Checkins>("Checkins", init<list>())
+        .def("compute", &Checkins::compute)
+    ;
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/boost-py/timeit.py	Sun Jun 02 06:53:16 2013 +0200
@@ -0,0 +1,18 @@
+from datetime import datetime
+from hello_ext import Checkins
+from math import pow
+
+amount = 2000
+mult = 1
+
+for i in range(8):
+    lst = [[1,2,3], [4,2,1], [1,4,2,5,6], [1,2,3,4,5], [1,2,3,4,5], [1,2,3,4], [1,5,2], [1,2,3], [1,2,3,4], [1,2,3,4], [1,2,3,4,5]]*mult
+    test = Checkins(lst)
+    now = datetime.now()
+
+    for i in range(amount):
+        res = test.compute(1)
+
+    speed = amount/(datetime.now()-now).total_seconds()
+    print mult*11, speed, speed*mult
+    mult = mult*2