Merge branch 'fix_aptcache_usage' into 'master'

Create class AptCache

- This class uses the apt.Cache, but verify if the package exists on xapian

See merge request !36

apprecommender/apt_cache.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+
+import apt
+import xapian
+
+
+class AptCache:
+
+    DEFAULT_AXI_PATH = "/var/lib/apt-xapian-index/index"
+
+    def __init__(self):
+        self.axi = xapian.Database(AptCache.DEFAULT_AXI_PATH)
+
+        self.cache = apt.Cache()
+
+    def __getitem__(self, pkg_name):
+        return self.get(pkg_name)
+
+    def __contains__(self, pkg_name):
+        return self.xapian_has_pkg(pkg_name) and pkg_name in self.cache
+
+    def get(self, pkg_name):
+        if self.xapian_has_pkg(pkg_name):
+            return self.cache[pkg_name]
+
+        raise KeyError("The cache has no package named '{}'".format(pkg_name))
+
+    def xapian_has_pkg(self, pkg_name):
+        term = 'XP' + pkg_name
+        return self.axi.get_termfreq(term) > 0L

apprecommender/decider.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python
 
-import apt
 import commands
 import re
 import xapian
 
+from apprecommender.apt_cache import AptCache
+
 from sklearn.feature_extraction.stop_words import ENGLISH_STOP_WORDS
 
 
@@ -21,7 +22,7 @@ class PkgInitDecider():
                         'fonts', 'png', 'core', 'default'}
 
     def __init__(self):
-        self.cache = apt.Cache()
+        self.cache = AptCache()
         self.user_role_programs = self.get_user_role_programs()
 
     def is_in_apt_cache(self, pkg):
@@ -195,7 +196,7 @@ def __init__(self, reverse_dependencies, user_installed_pkgs):
         self.reverse_dependencies = reverse_dependencies
         self.pkg_init_decider = PkgInitDecider()
         self.pkg_match_decider = PkgMatchDecider(user_installed_pkgs)
-        self.cache = apt.Cache()
+        self.cache = AptCache()
 
     def __call__(self, xapian_document):
         """

apprecommender/initialize.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 
-import apt
 import commands
 import data
 import datetime
@@ -24,7 +23,6 @@ class Initialize:
 
     def __init__(self):
         self.config = Config()
-        self.cache = apt.Cache()
         self.pkg_init_decider = PkgInitDecider()
 
     def get_tags(self):

apprecommender/main/collect_user_data.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 
-import apt
 import binascii
 import commands
 import datetime as dt
@@ -12,14 +11,15 @@
 import time
 import xapian
 
-from apprecommender.main.app_recommender import AppRecommender
+from apprecommender.apt_cache import AptCache
 from apprecommender.config import Config
 from apprecommender.data import get_user_installed_pkgs
 from apprecommender.data_classification import get_alternative_pkg
+from apprecommender.main.app_recommender import AppRecommender
+from apprecommender.main.ml_cross_validation import ml_cross_validation
 from apprecommender.ml.data import MachineLearningData
 from apprecommender.ml.pkg_time import PkgTime
 from apprecommender.utils import print_progress_bar
-from apprecommender.main.ml_cross_validation import ml_cross_validation
 
 LOG_PATH = os.path.expanduser('~/app_recommender_log')
 SUFIX = dt.datetime.now().strftime('%Y%m%d%H%M')
@@ -210,7 +210,7 @@ def collect_user_preferences():
 
     message_error = "\nPlease use digits 1-4 to rank a package: "
 
-    apt_cache = apt.Cache()
+    apt_cache = AptCache()
     for i in range(len(all_recommendations)):
         pkg = all_recommendations[i]
         pkg_description = apt_cache[pkg].versions[0].description

apprecommender/ml/bag_of_words.py

@@ -1,10 +1,10 @@
 import os
 import pickle
-from apt import Cache
 
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.naive_bayes import GaussianNB
 
+from apprecommender.apt_cache import AptCache
 from apprecommender.config import Config
 from apprecommender.ml.data import MachineLearningData
 
@@ -134,7 +134,7 @@ def save_pkgs_features(self, path, pkgs_list, features_array,
             pickle.dump(pkgs_classification, bow_pkgs_classification)
 
     def train_model(self, pkgs_list, axi, save_files=True):
-        cache = Cache()
+        cache = AptCache()
         ml_data = MachineLearningData()
 
         pkgs_description, pkg_classification = self.prepare_data(

apprecommender/ml/data.py

@@ -1,14 +1,14 @@
 from os import path
 from os import makedirs
 
-import apt
+import Stemmer
 import pickle
 import xapian
-import Stemmer
 
-from apprecommender.ml.pkg_time import PkgTime
+from apprecommender.apt_cache import AptCache
 from apprecommender.config import Config
 from apprecommender.decider import FilterTag, FilterDescription
+from apprecommender.ml.pkg_time import PkgTime
 
 
 class MachineLearningData():
@@ -41,7 +41,7 @@ def create_data(self, labels):
 
         pkgs = self.get_pkgs_classification(labels)
 
-        cache = apt.Cache()
+        cache = AptCache()
 
         terms_name = self.get_terms_for_all_pkgs(cache, pkgs.keys())
         debtags_name = self.get_debtags_for_all_pkgs(self.axi, pkgs.keys())

apprecommender/recommender.py

@@ -19,7 +19,6 @@
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
-import apt
 import heapq
 import inspect
 import logging
@@ -32,6 +31,8 @@
 from operator import attrgetter
 
 import apprecommender.strategy
+
+from apprecommender.apt_cache import AptCache
 from apprecommender.config import Config
 
 
@@ -46,7 +47,7 @@ def __init__(self, item_score, ranking=0, limit=0, user_profile=None):
         self.item_score = item_score
         self.size = len(item_score)
         self.limit = limit
-        self.cache = apt.Cache()
+        self.cache = AptCache()
         self.pkg_descriptions = {}
 
         if ranking:

apprecommender/strategy.py

@@ -20,7 +20,6 @@
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
-import apt
 import collections
 import logging
 import operator
@@ -34,6 +33,7 @@
 
 from abc import ABCMeta, abstractmethod
 
+from apprecommender.apt_cache import AptCache
 from apprecommender.config import Config
 from apprecommender.decider import (PkgMatchDecider,
                                     PkgReverseDependeciesDecider)
@@ -145,7 +145,7 @@ def __init__(self, content, profile_size):
         self.content = content
         self.description = 'Package-reference'
         self.profile_size = profile_size
-        self.cache = apt.Cache()
+        self.cache = AptCache()
         self.pkgs_regex = re.compile(r'^\s+(?:\|)?(.+)$', re.MULTILINE)
 
     def get_reverse_dependencies_pkgs(self, reference_pkgs):
@@ -209,7 +209,7 @@ def __init__(self, content, profile_size, suggestion_size=200):
         self.description = 'Machine-learning'
         self.profile_size = profile_size
         self.suggestion_size = suggestion_size
-        self.cache = apt.Cache()
+        self.cache = AptCache()
         self.ml_data = MachineLearningData()
         self.axi = xapian.Database(XAPIAN_DATABASE_PATH)
 

apprecommender/tests/test_ml/test_pkg_classification.py

@@ -1,19 +1,19 @@
 #!/usr/bin/env python
 
-import apt
 import unittest
 import xapian
 
 from mock import patch
 
+from apprecommender.apt_cache import AptCache
 from apprecommender.ml.data import MachineLearningData
 
 
 class PkgClassificationTests(unittest.TestCase):
 
     def setUp(self):
         self.ml_data = MachineLearningData()
-        self.cache = apt.Cache()
+        self.cache = AptCache()
 
     def test_get_pkg_debtags(self):
         vim_debtags = ['devel::editor', 'implemented-in::c',

apprecommender/user.py

@@ -20,7 +20,6 @@
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
-import apt
 import commands
 import datetime
 import glob
@@ -33,11 +32,12 @@
 
 import apprecommender.data as data
 
-from apprecommender.error import Error
-from apprecommender.singleton import Singleton
+from apprecommender.apt_cache import AptCache
+from apprecommender.config import Config
 from apprecommender.decider import (FilterTag, FilterDescription,
                                     FilterTag_or_Description)
-from apprecommender.config import Config
+from apprecommender.error import Error
+from apprecommender.singleton import Singleton
 
 
 class DemographicProfile(Singleton):
@@ -257,7 +257,7 @@ def maximal_pkg_profile(self):
         Return list of packages that are not dependence of any other package in
         the list.
         """
-        cache = apt.Cache()
+        cache = AptCache()
         old_profile_size = len(self.pkg_profile)
 
         for p in self.pkg_profile[:]:  # iterate list copy