Release 4.0.0beta

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2
 jobs:
   build:
     docker:
-      - image: circleci/python:2.7
+      - image: circleci/python:3.7.4
 
     working_directory: ~/gensim
 
@@ -18,16 +18,20 @@ jobs:
             sudo apt-get -yq update
             sudo apt-get -yq remove texlive-binaries --purge
             sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install dvipng texlive-latex-base texlive-latex-extra texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended latexmk
+            sudo apt-get -yq install build-essential python3.7-dev
 
       - run:
           name: Basic installation (tox)
           command: |
-            python -m virtualenv venv
+            python3.7 -m virtualenv venv
             source venv/bin/activate
-            pip install tox
+            pip install tox --progress-bar off
 
       - run:
           name: Build documentation
+          environment:
+            TOX_PARALLEL_NO_SPINNER: 1
+            TOX_PIP_OPTS: --progress-bar=off
           command: |
             source venv/bin/activate
             tox -e compile,docs -vv

.travis.yml

@@ -8,32 +8,46 @@ cache:
   - $HOME/.pip-cache
 dist: trusty
 language: python
+env:
+  TOX_PARALLEL_NO_SPINNER: 1
 
 
 matrix:
   include:
-    - python: '2.7'
-      env: TOXENV="flake8,flake8-docs"
-
     - python: '3.6'
       env: TOXENV="flake8,flake8-docs"
 
-    - python: '2.7'
-      env: TOXENV="py27-linux"
-
-    - python: '3.5'
-      env: TOXENV="py35-linux"
-
-    - python: '3.6'
-      env: TOXENV="py36-linux"
+    - python: '3.8'
+      env:
+        - TOXENV="py38-linux"
+      dist: bionic
 
     - python: '3.7'
       env:
         - TOXENV="py37-linux"
-        - BOTO_CONFIG="/dev/null"
+      # The following two lines used to be necessary because Travis left files lying around in ~/.aws/,
+      # messing up our tests. Now fixed since https://github.com/travis-ci/travis-ci/issues/7940
+        # - BOTO_CONFIG="/dev/null"
+      #sudo: true
       dist: xenial
-      sudo: true
+
+    - python: '3.6'
+      env: TOXENV="py36-linux"
+
+
+install:
+  - pip install tox
+  - sudo apt-get install -y gdb
+
+
+before_script:
+  - ulimit -c unlimited -S  # enable core dumps
 
 
-install: pip install tox
 script: tox -vv
+
+
+after_failure:
+  - pwd
+  - COREFILE=$(find . -maxdepth 1 -name "core*" | head -n 1)
+  - if [[ -f "$COREFILE" ]]; then EXECFILE=$(gdb -c "$COREFILE" -batch | grep "Core was generated" | tr -d "\`" | cut -d' ' -f5); file "$COREFILE"; gdb -c "$COREFILE" "$EXECFILE" -x continuous_integration/debug.gdb -batch; fi

CONTRIBUTING.md

@@ -22,8 +22,8 @@ Also, please check the [Gensim FAQ](https://github.com/RaRe-Technologies/gensim/
 6. Check that everything's OK in your branch:
    - Check it for PEP8: `tox -e flake8`
    - Build its documentation (works only for MacOS/Linux): `tox -e docs` (documentation stored in `docs/src/_build`)
-   - Run unit tests: `tox -e py{version}-{os}`, for example `tox -e py27-linux` or `tox -e py36-win` where
-      - `{version}` is one of `27`, `35`, `36`
+   - Run unit tests: `tox -e py{version}-{os}`, for example `tox -e py35-linux` or `tox -e py36-win` where
+      - `{version}` is one of `35`, `36`
       - `{os}` is either `win` or `linux`
 7. Add files, commit and push: `git add ... ; git commit -m "my commit message"; git push origin my-feature`
 8. [Create a PR](https://help.github.com/articles/creating-a-pull-request/) on Github. Write a **clear description** for your PR, including all the context and relevant information, such as:

HACKTOBERFEST.md

@@ -0,0 +1,35 @@
+# :pizza: Hacktoberfest 2019 :beer:
+
+It's that time of the year again!
+[Hacktoberfest](https://hacktoberfest.digitalocean.com) is here, and `gensim` needs **your** help.
+We've prepared a list of good issues to work on: [gensim hacktoberfest issues](https://github.com/RaRe-Technologies/gensim/labels/hacktoberfest).
+
+If the learning curve for `gensim` is a bit steep, give the [smart_open](https://github.com/RaRe-Technologies/smart_open) repository a try.
+`smart_open` is an important dependency of `gensim`: it performs file I/O over a variety of protocols and formats.
+There's also a list of Hacktoberfest-friendly issues to work on: [smart_open hacktoberfest issues](https://github.com/RaRe-Technologies/smart_open/labels/hacktoberfest).
+
+Of course, we welcome contributions on any of the existing issues, not just the ones labeled `hacktoberfest`.
+If the issue is simple & quick, you can just submit your PR, with a proper reference to the issue it addresses.
+If the issue requires a little more work, but you have a good idea of how to proceed & know when you'll be submitting some initial work, please post a short note about your plans to the issue, or a "work-in-progress" ("[WIP]") pull-request indicating work is underway, to help avoid wasted duplicate work.
+
+Furthermore, we also welcome contributions not connected to an existing issue.
+This includes things like fixing typos in documentation, docstrings, etc.
+If you make such contributions, please make the motivation behind the contribution clear.
+You could start such a contribution with a new pull-request, or if you think it requires other discussion beforehand, as a separate new issue.
+Please avoid making innocuous changes without sufficient motivation (e.g. changing code formatting, etc).
+
+## Before Contributing
+
+Check out the following:
+
+- [First-time contributors guide](https://github.com/firstcontributions/first-contributions): if this is your first time contributing on GitHub.
+- [Hacktoberfest rules](https://hacktoberfest.digitalocean.com/faq#rules): read this in full
+- [Developer page](https://github.com/RaRe-Technologies/gensim/wiki/Developer-page) on our Wiki: for the git flow, code style, etc.
+
+## Questions
+
+If you have a general question about Gensim, please ask on the [mailing list](https://groups.google.com/forum/#!forum/gensim).
+If you have a question a about a specific issue or PR, just ask there directly, and we'll get back to you as soon as we can.
+Otherwise, ping @mpenkov on [Twitter](https://twitter.com/mpenkov) or [Telegram](https://t.me/mpenkov).
+
+Happy Hacking!!

ISSUE_TEMPLATE.md

@@ -22,6 +22,7 @@ Please provide the output of:
 ```python
 import platform; print(platform.platform())
 import sys; print("Python", sys.version)
+import struct; print("Bits", 8 * struct.calcsize("P"))
 import numpy; print("NumPy", numpy.__version__)
 import scipy; print("SciPy", scipy.__version__)
 import gensim; print("gensim", gensim.__version__)

MANIFEST.in

@@ -28,8 +28,6 @@ include gensim/models/fasttext_inner.pxd
 include gensim/models/fasttext_corpusfile.cpp
 include gensim/models/fasttext_corpusfile.pyx
 
-include gensim/models/_utils_any2vec.c
-include gensim/models/_utils_any2vec.pyx
 include gensim/corpora/_mmreader.c
 include gensim/corpora/_mmreader.pyx
 include gensim/_matutils.c

README.md

@@ -1,22 +1,31 @@
 gensim – Topic Modelling in Python
 ==================================
 
+<!--
+The following image URLs are obfuscated = proxied and cached through
+Google because of Github's proxying issues. See:
+https://github.com/RaRe-Technologies/gensim/issues/2805
+-->
+
 [![Build Status](https://travis-ci.org/RaRe-Technologies/gensim.svg?branch=develop)](https://travis-ci.org/RaRe-Technologies/gensim)
 [![GitHub release](https://img.shields.io/github/release/rare-technologies/gensim.svg?maxAge=3600)](https://github.com/RaRe-Technologies/gensim/releases)
-[![Conda-forge Build](https://anaconda.org/conda-forge/gensim/badges/version.svg)](https://anaconda.org/conda-forge/gensim)
-[![Wheel](https://img.shields.io/pypi/wheel/gensim.svg)](https://pypi.python.org/pypi/gensim)
+[![Downloads](https://img.shields.io/pypi/dm/gensim?color=blue)](https://pepy.tech/project/gensim/month)
 [![DOI](https://zenodo.org/badge/DOI/10.13140/2.1.2393.1847.svg)](https://doi.org/10.13140/2.1.2393.1847)
-[![Mailing List](https://img.shields.io/badge/-Mailing%20List-brightgreen.svg)](https://groups.google.com/forum/#!forum/gensim)
-[![Gitter](https://img.shields.io/badge/gitter-join%20chat%20%E2%86%92-09a3d5.svg)](https://gitter.im/RaRe-Technologies/gensim)
-[![Follow](https://img.shields.io/twitter/follow/gensim_py.svg?style=social&label=Follow)](https://twitter.com/gensim_py)
-
-
+[![Mailing List](https://img.shields.io/badge/-Mailing%20List-blue.svg)](https://groups.google.com/forum/#!forum/gensim)
+[![Follow](https://img.shields.io/twitter/follow/gensim_py.svg?style=social&style=flat&logo=twitter&label=Follow&color=blue)](https://twitter.com/gensim_py)
 
 Gensim is a Python library for *topic modelling*, *document indexing*
 and *similarity retrieval* with large corpora. Target audience is the
 *natural language processing* (NLP) and *information retrieval* (IR)
 community.
 
+<!--
+## :pizza: Hacktoberfest 2019 :beer:
+
+We are accepting PRs for Hacktoberfest!
+See [here](HACKTOBERFEST.md) for details.
+-->
+
 Features
 --------
 
@@ -40,13 +49,6 @@ If this feature list left you scratching your head, you can first read
 more about the [Vector Space Model] and [unsupervised document analysis]
 on Wikipedia.
 
-Support
-------------
-
-Ask open-ended or research questions on the [Gensim Mailing List](https://groups.google.com/forum/#!forum/gensim).
-
-Raise bugs on [Github](https://github.com/RaRe-Technologies/gensim/blob/develop/CONTRIBUTING.md) but **make sure you follow the [issue template](https://github.com/RaRe-Technologies/gensim/blob/develop/ISSUE_TEMPLATE.md)**. Issues that are not bugs or fail to follow the issue template will be closed without inspection.
-
 Installation
 ------------
 
@@ -60,23 +62,23 @@ NumPy. This is optional, but using an optimized BLAS such as [ATLAS] or
 magnitude. On OS X, NumPy picks up the BLAS that comes with it
 automatically, so you don’t need to do anything special.
 
-The simple way to install gensim is:
+Install the latest version of gensim:
 
-    pip install -U gensim
+```bash
+    pip install --upgrade gensim
+```
 
 Or, if you have instead downloaded and unzipped the [source tar.gz]
-package, you’d run:
+package:
 
-    python setup.py test
+```bash
     python setup.py install
+```
 
-For alternative modes of installation (without root privileges,
-development installation, optional install features), see the
-[documentation].
+For alternative modes of installation, see the [documentation].
 
-This version has been tested under Python 2.7, 3.5 and 3.6. Gensim’s github repo is hooked
-against [Travis CI for automated testing] on every commit push and pull
-request. Support for Python 2.6, 3.3 and 3.4 was dropped in gensim 1.0.0. Install gensim 0.13.4 if you *must* use Python 2.6, 3.3 or 3.4. Support for Python 2.5 was dropped in gensim 0.10.0; install gensim 0.9.1 if you *must* use Python 2.5). 
+Gensim is being [continuously tested](https://travis-ci.org/RaRe-Technologies/gensim) under Python 3.6, 3.7 and 3.8.
+Support for Python 2.7 was dropped in gensim 4.0.0 – install gensim 3.8.3 if you must use Python 2.7.
 
 How come gensim is so fast and memory efficient? Isn’t it pure Python, and isn’t Python slow and greedy?
 --------------------------------------------------------------------------------------------------------
@@ -98,22 +100,27 @@ Documentation
 
 -   [QuickStart]
 -   [Tutorials]
--   [Tutorial Videos]
 -   [Official API Documentation]
 
-  [QuickStart]: https://github.com/RaRe-Technologies/gensim/blob/develop/docs/notebooks/gensim%20Quick%20Start.ipynb
-  [Tutorials]: https://github.com/RaRe-Technologies/gensim/blob/develop/tutorials.md#tutorials
-  [Tutorial Videos]: https://github.com/RaRe-Technologies/gensim/blob/develop/tutorials.md#videos
+  [QuickStart]: https://radimrehurek.com/gensim/auto_examples/core/run_core_concepts.html
+  [Tutorials]: https://radimrehurek.com/gensim/auto_examples/
   [Official Documentation and Walkthrough]: http://radimrehurek.com/gensim/
   [Official API Documentation]: http://radimrehurek.com/gensim/apiref.html
-
+
+Support
+-------
+
+Ask open-ended or research questions on the [Gensim Mailing List](https://groups.google.com/forum/#!forum/gensim).
+
+Raise bugs on [Github](https://github.com/RaRe-Technologies/gensim/blob/develop/CONTRIBUTING.md) but **make sure you follow the [issue template](https://github.com/RaRe-Technologies/gensim/blob/develop/ISSUE_TEMPLATE.md)**. Issues that are not bugs or fail to follow the issue template will be closed without inspection.
+
 ---------
 
 Adopters
 --------
 
 | Company | Logo | Industry | Use of Gensim |
-|---------|------|----------|---------------|                          
+|---------|------|----------|---------------|
 | [RARE Technologies](http://rare-technologies.com) | ![rare](docs/src/readme_images/rare.png) | ML & NLP consulting | Creators of Gensim – this is us! |
 | [Amazon](http://www.amazon.com/) |  ![amazon](docs/src/readme_images/amazon.png) | Retail |  Document similarity. |
 | [National Institutes of Health](https://github.com/NIHOPA/pipeline_word2vec) | ![nih](docs/src/readme_images/nih.png) | Health | Processing grants and publications with word2vec. |
@@ -162,8 +169,8 @@ BibTeX entry:
   [Talentpair]: https://avatars3.githubusercontent.com/u/8418395?v=3&s=100
   [citing gensim in academic papers and theses]: https://scholar.google.cz/citations?view_op=view_citation&hl=en&user=9vG_kV0AAAAJ&citation_for_view=9vG_kV0AAAAJ:u-x6o8ySG0sC
 
-  
-  
+
+
   [documentation and Jupyter Notebook tutorials]: https://github.com/RaRe-Technologies/gensim/#documentation
   [Vector Space Model]: http://en.wikipedia.org/wiki/Vector_space_model
   [unsupervised document analysis]: http://en.wikipedia.org/wiki/Latent_semantic_indexing

azure-pipelines.yml

@@ -0,0 +1,29 @@
+pool:
+  vmImage: 'vs2017-win2016'
+
+strategy:
+  matrix:
+    py36:
+      python.version: '3.6'
+      TOXENV: "py36-win"
+    py37:
+      python.version: '3.7'
+      TOXENV: "py37-win"
+    py38:
+      python.version: '3.8'
+      TOXENV: "py38-win"
+
+steps:
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: '$(python.version)'
+  displayName: 'Use Python $(python.version)'
+
+- script: |
+    python -m pip install --upgrade pip
+    python -m pip install tox
+  displayName: 'Install tox'
+
+- script: |
+    tox -vv
+  displayName: 'Testing'