Merge pull request #146 from FreeJournal/develop

Final Iteration

.travis.yml

@@ -15,7 +15,7 @@ install:
 
 # command to run tests, e.g. python setup.py test
 script:
-  - coverage run --omit=*/python?.?/*,*/site-packages/*,*__init__*,test_* -m unittest discover unittests
+  - coverage run --omit=*/python?.?/*,*/site-packages/*,*__init__*,test_*,*Freenet* -m unittest discover unittests
 
 after_success:
   coveralls

README.md

@@ -17,12 +17,11 @@ The first step to installing FreeJournal is to ensure you have:
 If you are on Windows, you can still browse one of many existing FreeJournal nodes.
 Consult a list on our website.
 
-Run python install.py in bitmessage/ and in freenet/
-to install the required prerequisites, then modify config.py to 
-change your local node settings.
-
 To install database, documentation, and other library requirements, run
-``pip install -r requirements.txt`` on the root directory.
+``sudo pip install -r requirements.txt`` on the root directory.
+
+After this, you should perform the required Freenet and Bitmessage setup using:
+``sudo ./freejournal_cli.py install all`` on a Debian-compatible distribution.
 
 ## Generating Documentation
 
@@ -36,20 +35,27 @@ To generate API documentation, simply run
 
 Modify the following example commands to create and publish a collection:
 ```
-./freejournal-cli putcollection whee 1,2,3 "This is a TEST" "nothing to see here" "nothing,to" btc123
+./freejournal_cli.py putcollection whee 1,2,3 "This is a TEST" "nothing to see here" "nothing,to" btc123
 Collection inserted with address/ID [BM-2cVBBDezMcgoAHMkNzMswkc3xZRMFFvKeV
-./freejournal-cli publishcollection whee BM-2cVBBDezMcgoAHMkNzMswkc3xZRMFFvKeV
+./freejournal_cli.py publishcollection whee BM-2cVBBDezMcgoAHMkNzMswkc3xZRMFFvKeV
 ```
 
-... more coming soon
+For more command usage instructions, run ``./freejournal_cli.py`` with no arguments.
 
 ## Running FreeJournal Unit Tests
 
-Simply `python run_tests.py` in the root directory after installing the required
-prerequisites.
+Simply ``coverage run --omit=*/python?.?/*,*/site-packages/*,*__init__*,test_*,*Freenet* -m unittest discover unittests`` 
+in the root directory after installing the required prerequisites to run all tests with coverage.
+
+## Running FreeJournal - The Network
+
+To start pulling and pushing collections to and from the network, use the following two commands:
+
+``./freejournal_cli.py keepalive`` and ``./freejournal_cli.py listen``
+
 
 ## Running FreeJournal - Web interface
 
-After configuring the web interface in config.py, use `./freejournal-cli webapp` to run an insecure
+After configuring the web interface in config.py, use `./freejournal_cli.py webapp` to run an insecure
  development server.  Hardened deployments should use the advanced deployment guides on the Flask 
 website.

async/__init__.py

@@ -36,11 +36,11 @@ def loop():
     return decorator
 
 
-def wait_for_interrupt(func, stop_event):
+def wait_for_interrupt(func, stop_event, args=[]):
     while True:
         try:
             sleep(10)
         except (KeyboardInterrupt, SystemExit):
             stop_event.set()
-            func()
+            func(*args)
             exit(0)

bitmessage/bitmessage_keepalive.py

@@ -5,7 +5,7 @@
 from datetime import datetime
 
 
-def rebroadcast(collection):
+def rebroadcast(collection, testing_mode=False):
     """
     Rebroadcast the collection to the Main Channel
     :param collection: the collection object
@@ -21,14 +21,18 @@ def rebroadcast(collection):
     controller = Controller()
     print("Rebroadcasting collection: " + collection.title)
 
-    success = controller.publish_collection(collection, MAIN_CHANNEL_ADDRESS, from_address)
+    if testing_mode:
+        success = controller.publish_collection(collection, from_address, from_address)
+    else:
+        success = controller.publish_collection(collection, MAIN_CHANNEL_ADDRESS, from_address)
+
     if not success:
         return False
 
     return True
 
 
-def find_old_collections(keepalive_constant):
+def find_old_collections(keepalive_constant, testing_mode=False):
     """
     The main keep alive function that searches the cache
     for older collections that should be rebroadcasted to
@@ -45,7 +49,11 @@ def find_old_collections(keepalive_constant):
         age = today - collection.latest_broadcast_date
         if age.days >= keepalive_constant:
             collection.latest_broadcast_date = datetime.today()
-            success = rebroadcast(collection)
+
+            if testing_mode:
+                success = rebroadcast(collection, testing_mode=True)
+            else:
+                success = rebroadcast(collection)
 
             if success:
                 print("Updating collection in cache")

bitmessage/bitmessage_listener.py

@@ -1,18 +1,35 @@
 from controllers.controller import Controller
 from config import MAIN_CHANNEL_ADDRESS, LISTEN_PERIOD
-from async import repeat_periodic, wait_for_interrupt
+from async import repeat_periodic, wait_for_interrupt, run_as_thread
+from time import sleep
 
+@run_as_thread
+def wait_for_join(controller):
+    controller.join_downloads()
 
-def exit_func():
+
+def exit_func(controller):
     print ""
-    print "exiting."
+    if controller.alive_downloads():
+        print "Downloads in progress, please wait for them to finish. Press ctrl+c again to cancel downloads."
+        try:
+            t = wait_for_join(controller)
+            while t.is_alive():
+                sleep(0.1)
+            print "Exited safely."
+        except (KeyboardInterrupt, SystemExit):
+            print ""
+            print "Cancelling downloads and exiting."
+            exit(1)
+    else:
+        print "Exiting listener."
 
 
 def get_collections():
     """Get collections from the main channel"""
 
     collection_handler = Controller()
-    wait_for_interrupt(exit_func, do_import(collection_handler))
+    wait_for_interrupt(exit_func, do_import(collection_handler), args=[collection_handler])
 
 
 @repeat_periodic(LISTEN_PERIOD)

bitmessage/installfiles/keys.dat

@@ -43,11 +43,11 @@ apiusername = username
 apipassword = password
 daemon = true
 
-[BM-2cVqkJCqk9RTapvmVwEd2mrKUnzLoAcHLx]
-label = Free Journal Main Channel
+[BM-2cXExRht8eiB7anRv8Rc3TvSTm4GcJ2Txx]
+label = FreeJournal Main Channel
 enabled = true
 decoy = false
 noncetrialsperbyte = 1000
 payloadlengthextrabytes = 1000
-privsigningkey = 5J15mbNV5dWEvLcdVf2wua2Xn3Hn73qnNuacVcYt3zo8CKeSwCE
-privencryptionkey = 5JBZS85dLU8QTK5gPxH3YGnEgiGyaB8RjQ9J6yEuTsAtSTrbZYR
+privsigningkey = 5J3WLPnGPcKxsx8Vk24WH4VLzaEZ3xLTCjsc1xaha9HdZPbgWbT
+privencryptionkey = 5KbHD145pnNGcTnZ3kSL3RzHxc3zoUuBWLS8Z51w1dRkj9JVWZf

cache/cache.py

@@ -35,6 +35,15 @@ def get_all_collections(self):
         """
         return self.session.query(Collection).order_by(Collection.creation_date.desc())
 
+    def get_collections_paginated(self, limit, offset):
+        """
+        Get a few collections, selecting using a limit and offset
+        :param limit: maximum number of collections to return
+        :param offset: how many collections to skip
+        :return: list of collections, ordered by date added
+        """
+        return self.session.query(Collection).order_by(Collection.creation_date.desc()).offset(offset).limit(limit)
+
     def get_collection_with_address(self, address):
         """
         Retreive a specific collection as it's stored locally
@@ -61,6 +70,19 @@ def get_keyword_by_id(self, cur_id):
             pass
         return row
 
+    def get_keyword_by_name(self, name):
+        """
+        Retrieve a specific keyword by it's name
+        :param name: The name of the requested keyword
+        :return: The Keyword object if in the cache, None otherwise
+        """
+        row = None
+        try:
+            row = self.session.query(Keyword).filter(Keyword.name == name).one()
+        except NoResultFound:
+            pass
+        return row
+
     def get_signature_by_address(self, address):
         """
         Retrieve a specific signature by it's collection address
@@ -107,10 +129,10 @@ def insert_new_document(self,document):
         collection = self.session.query(Collection).filter_by(address = document.collection_address).first()
         self.insert_new_document_in_collection(document, collection)
 
-    #Note, if this is called manually(and not via cli/api) the collection root hash will not be updated
     def insert_new_document_in_collection(self,document,collection):
         """
         Insert a new document associated with an existing collection into local storage
+        WARNING: if this is called manually(and not via cli/api) the collection root hash will not be updated
         :param document: Document object to insert into local storage
         """
         self.session.add(document)
@@ -134,5 +156,13 @@ def remove_collection(self, collection):
         self.session.delete(collection)
         self.session.commit()
 
+    def remove_document(self, document):
+        """
+        Remove a document from a collection from local storage
+        :param document: Document object to remove from local storage
+        """
+        self.session.delete(document)
+        self.session.commit()
+
     def close(self):
         self.session.close()

config.py

@@ -25,15 +25,17 @@
 
 ''' Don't change this unless the main channel address is changed in the future '''
 
-MAIN_CHANNEL_ADDRESS = "BM-2cVqkJCqk9RTapvmVwEd2mrKUnzLoAcHLx"
-privsigningkey = "5J15mbNV5dWEvLcdVf2wua2Xn3Hn73qnNuacVcYt3zo8CKeSwCE"
-privencryptionkey = "5JBZS85dLU8QTK5gPxH3YGnEgiGyaB8RjQ9J6yEuTsAtSTrbZYR"
+MAIN_CHANNEL_ADDRESS = "BM-2cXExRht8eiB7anRv8Rc3TvSTm4GcJ2Txx"
 
 ''' Directory to store documents that are downloaded
 (Make sure to keep the last '/' at the end of the path)'''
 DOCUMENT_DIRECTORY_PATH = "~/Documents/"
 
+''' Webapp settings'''
 # Set to true at your own risk
 WEBAPP_DEBUG = True
 WEBAPP_PORT  = 5000
+INDEX_LIMIT = 10
+
+''' Listener settings'''
 LISTEN_PERIOD = 30