diff --git a/apprecommender/apt_cache.py b/apprecommender/apt_cache.py new file mode 100644 index 0000000..e3b5dcc --- /dev/null +++ b/apprecommender/apt_cache.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python + +import apt +import xapian + + +class AptCache: + + DEFAULT_AXI_PATH = "/var/lib/apt-xapian-index/index" + + def __init__(self): + self.axi = xapian.Database(AptCache.DEFAULT_AXI_PATH) + + self.cache = apt.Cache() + + def __getitem__(self, pkg_name): + return self.get(pkg_name) + + def __contains__(self, pkg_name): + return self.xapian_has_pkg(pkg_name) and pkg_name in self.cache + + def get(self, pkg_name): + if self.xapian_has_pkg(pkg_name): + return self.cache[pkg_name] + + raise KeyError("The cache has no package named '{}'".format(pkg_name)) + + def xapian_has_pkg(self, pkg_name): + term = 'XP' + pkg_name + return self.axi.get_termfreq(term) > 0L diff --git a/apprecommender/decider.py b/apprecommender/decider.py index 2e55db3..9830026 100644 --- a/apprecommender/decider.py +++ b/apprecommender/decider.py @@ -1,10 +1,11 @@ #!/usr/bin/env python -import apt import commands import re import xapian +from apprecommender.apt_cache import AptCache + from sklearn.feature_extraction.stop_words import ENGLISH_STOP_WORDS @@ -21,7 +22,7 @@ class PkgInitDecider(): 'fonts', 'png', 'core', 'default'} def __init__(self): - self.cache = apt.Cache() + self.cache = AptCache() self.user_role_programs = self.get_user_role_programs() def is_in_apt_cache(self, pkg): @@ -195,7 +196,7 @@ def __init__(self, reverse_dependencies, user_installed_pkgs): self.reverse_dependencies = reverse_dependencies self.pkg_init_decider = PkgInitDecider() self.pkg_match_decider = PkgMatchDecider(user_installed_pkgs) - self.cache = apt.Cache() + self.cache = AptCache() def __call__(self, xapian_document): """ diff --git a/apprecommender/initialize.py b/apprecommender/initialize.py index 04cd3de..fe4618d 100644 --- a/apprecommender/initialize.py +++ b/apprecommender/initialize.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -import apt import commands import data import datetime @@ -24,7 +23,6 @@ class Initialize: def __init__(self): self.config = Config() - self.cache = apt.Cache() self.pkg_init_decider = PkgInitDecider() def get_tags(self): diff --git a/apprecommender/main/collect_user_data.py b/apprecommender/main/collect_user_data.py index 92a29b2..761795d 100644 --- a/apprecommender/main/collect_user_data.py +++ b/apprecommender/main/collect_user_data.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -import apt import binascii import commands import datetime as dt @@ -12,14 +11,15 @@ import time import xapian -from apprecommender.main.app_recommender import AppRecommender +from apprecommender.apt_cache import AptCache from apprecommender.config import Config from apprecommender.data import get_user_installed_pkgs from apprecommender.data_classification import get_alternative_pkg +from apprecommender.main.app_recommender import AppRecommender +from apprecommender.main.ml_cross_validation import ml_cross_validation from apprecommender.ml.data import MachineLearningData from apprecommender.ml.pkg_time import PkgTime from apprecommender.utils import print_progress_bar -from apprecommender.main.ml_cross_validation import ml_cross_validation LOG_PATH = os.path.expanduser('~/app_recommender_log') SUFIX = dt.datetime.now().strftime('%Y%m%d%H%M') @@ -210,7 +210,7 @@ def collect_user_preferences(): message_error = "\nPlease use digits 1-4 to rank a package: " - apt_cache = apt.Cache() + apt_cache = AptCache() for i in range(len(all_recommendations)): pkg = all_recommendations[i] pkg_description = apt_cache[pkg].versions[0].description diff --git a/apprecommender/ml/bag_of_words.py b/apprecommender/ml/bag_of_words.py index f612412..704915c 100644 --- a/apprecommender/ml/bag_of_words.py +++ b/apprecommender/ml/bag_of_words.py @@ -1,10 +1,10 @@ import os import pickle -from apt import Cache from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.naive_bayes import GaussianNB +from apprecommender.apt_cache import AptCache from apprecommender.config import Config from apprecommender.ml.data import MachineLearningData @@ -134,7 +134,7 @@ def save_pkgs_features(self, path, pkgs_list, features_array, pickle.dump(pkgs_classification, bow_pkgs_classification) def train_model(self, pkgs_list, axi, save_files=True): - cache = Cache() + cache = AptCache() ml_data = MachineLearningData() pkgs_description, pkg_classification = self.prepare_data( diff --git a/apprecommender/ml/data.py b/apprecommender/ml/data.py index 7ecd636..61f864c 100644 --- a/apprecommender/ml/data.py +++ b/apprecommender/ml/data.py @@ -1,14 +1,14 @@ from os import path from os import makedirs -import apt +import Stemmer import pickle import xapian -import Stemmer -from apprecommender.ml.pkg_time import PkgTime +from apprecommender.apt_cache import AptCache from apprecommender.config import Config from apprecommender.decider import FilterTag, FilterDescription +from apprecommender.ml.pkg_time import PkgTime class MachineLearningData(): @@ -41,7 +41,7 @@ def create_data(self, labels): pkgs = self.get_pkgs_classification(labels) - cache = apt.Cache() + cache = AptCache() terms_name = self.get_terms_for_all_pkgs(cache, pkgs.keys()) debtags_name = self.get_debtags_for_all_pkgs(self.axi, pkgs.keys()) diff --git a/apprecommender/recommender.py b/apprecommender/recommender.py index f702393..087755d 100644 --- a/apprecommender/recommender.py +++ b/apprecommender/recommender.py @@ -19,7 +19,6 @@ along with this program. If not, see . """ -import apt import heapq import inspect import logging @@ -32,6 +31,8 @@ from operator import attrgetter import apprecommender.strategy + +from apprecommender.apt_cache import AptCache from apprecommender.config import Config @@ -46,7 +47,7 @@ def __init__(self, item_score, ranking=0, limit=0, user_profile=None): self.item_score = item_score self.size = len(item_score) self.limit = limit - self.cache = apt.Cache() + self.cache = AptCache() self.pkg_descriptions = {} if ranking: diff --git a/apprecommender/strategy.py b/apprecommender/strategy.py index c662277..8a47fe0 100644 --- a/apprecommender/strategy.py +++ b/apprecommender/strategy.py @@ -20,7 +20,6 @@ along with this program. If not, see . """ -import apt import collections import logging import operator @@ -34,6 +33,7 @@ from abc import ABCMeta, abstractmethod +from apprecommender.apt_cache import AptCache from apprecommender.config import Config from apprecommender.decider import (PkgMatchDecider, PkgReverseDependeciesDecider) @@ -145,7 +145,7 @@ def __init__(self, content, profile_size): self.content = content self.description = 'Package-reference' self.profile_size = profile_size - self.cache = apt.Cache() + self.cache = AptCache() self.pkgs_regex = re.compile(r'^\s+(?:\|)?(.+)$', re.MULTILINE) def get_reverse_dependencies_pkgs(self, reference_pkgs): @@ -209,7 +209,7 @@ def __init__(self, content, profile_size, suggestion_size=200): self.description = 'Machine-learning' self.profile_size = profile_size self.suggestion_size = suggestion_size - self.cache = apt.Cache() + self.cache = AptCache() self.ml_data = MachineLearningData() self.axi = xapian.Database(XAPIAN_DATABASE_PATH) diff --git a/apprecommender/tests/test_ml/test_pkg_classification.py b/apprecommender/tests/test_ml/test_pkg_classification.py index 90a433a..b1f15b0 100644 --- a/apprecommender/tests/test_ml/test_pkg_classification.py +++ b/apprecommender/tests/test_ml/test_pkg_classification.py @@ -1,11 +1,11 @@ #!/usr/bin/env python -import apt import unittest import xapian from mock import patch +from apprecommender.apt_cache import AptCache from apprecommender.ml.data import MachineLearningData @@ -13,7 +13,7 @@ class PkgClassificationTests(unittest.TestCase): def setUp(self): self.ml_data = MachineLearningData() - self.cache = apt.Cache() + self.cache = AptCache() def test_get_pkg_debtags(self): vim_debtags = ['devel::editor', 'implemented-in::c', diff --git a/apprecommender/user.py b/apprecommender/user.py index fb78873..2c183a5 100644 --- a/apprecommender/user.py +++ b/apprecommender/user.py @@ -20,7 +20,6 @@ along with this program. If not, see . """ -import apt import commands import datetime import glob @@ -33,11 +32,12 @@ import apprecommender.data as data -from apprecommender.error import Error -from apprecommender.singleton import Singleton +from apprecommender.apt_cache import AptCache +from apprecommender.config import Config from apprecommender.decider import (FilterTag, FilterDescription, FilterTag_or_Description) -from apprecommender.config import Config +from apprecommender.error import Error +from apprecommender.singleton import Singleton class DemographicProfile(Singleton): @@ -257,7 +257,7 @@ def maximal_pkg_profile(self): Return list of packages that are not dependence of any other package in the list. """ - cache = apt.Cache() + cache = AptCache() old_profile_size = len(self.pkg_profile) for p in self.pkg_profile[:]: # iterate list copy