Bump nixpkgs and python devs, improve out nix build

CHANGELOG.md

@@ -1,3 +1,5 @@
+  * 0.8.2:
+    - Fix: bump all dependencies (getting rid of vulnerable packages)
   * 0.8.1
     - Feature: allow merging in order of last-update time #149
   * 0.8.0

Makefile

@@ -1,37 +1,46 @@
 VERSION?=$$(git rev-parse --abbrev-ref HEAD)
 
-requirements_frozen.txt requirements.nix requirements_override.nix: requirements.txt
-	pypi2nix -V 3.6 -r $^
-
 .PHONY: all
-all: requirements_frozen.txt requirements.nix requirements_override.nix default.nix
-	nix-build -K .
+all: requirements_frozen.txt requirements.nix requirements_override.nix marge-bot dockerize
+
+.PHONY: marge-bot
+marge-bot:
+	nix-build --keep-failed --attr marge-bot default.nix
 
 .PHONY: clean
 clean:
 	rm -rf .cache result requirements_frozen.txt
 
+.PHONY: bump
+bump: bump-requirements bump-sources
+
+.PHONY: bump-sources
+bump-sources:
+	nix-shell --run niv update
+
 .PHONY: bump-requirements
 bump-requirements: clean requirements_frozen.txt
 
-.PHONY: dockerize
-dockerize: dockerize.nix
-	docker load --input $$(nix-build dockerize.nix)
+requirements_frozen.txt requirements.nix requirements_override.nix: requirements.txt
+	pypi2nix -V 3.6 -r $^
 
+.PHONY: dockerize
+dockerize:
+	docker load --input $$(nix-build --attr docker-image default.nix)
 
 .PHONY: docker-push
 docker-push:
 	if [ -n "$$DOCKER_USERNAME" -a -n "$$DOCKER_PASSWORD" ]; then \
-	  docker login -u "$${DOCKER_USERNAME}" -p "$${DOCKER_PASSWORD}"; \
+		docker login -u "$${DOCKER_USERNAME}" -p "$${DOCKER_PASSWORD}"; \
 	else \
-	  docker login; \
+		docker login; \
 	fi
 	docker tag smarkets/marge-bot:$$(cat version) smarkets/marge-bot:$(VERSION)
 	if [ "$(VERSION)" = "$$(cat version)" ]; then \
-	  docker tag smarkets/marge-bot:$$(cat version) smarkets/marge-bot:latest; \
-	  docker tag smarkets/marge-bot:$$(cat version) smarkets/marge-bot:stable; \
-	  docker push smarkets/marge-bot:stable; \
-	  docker push smarkets/marge-bot:latest; \
+		docker tag smarkets/marge-bot:$$(cat version) smarkets/marge-bot:latest; \
+		docker tag smarkets/marge-bot:$$(cat version) smarkets/marge-bot:stable; \
+		docker push smarkets/marge-bot:stable; \
+		docker push smarkets/marge-bot:latest; \
 	fi
 	docker push smarkets/marge-bot:$(VERSION)
 	# for backwards compatibility push to previous location

default.nix

@@ -1,3 +1,6 @@
-let pkgs = import ./pinnedNixpkgs.nix;
-in
-pkgs.callPackage ./marge.nix {}
+let sources = import ./nix/sources.nix; in
+with import sources.nixpkgs {};
+{
+  marge-bot = callPackage ./marge.nix {};
+  docker-image = callPackage ./dockerize.nix {};
+}

dockerize.nix

@@ -1,20 +1,31 @@
 { pkgs ? import ./pinnedNixpkgs.nix }:
-let callPackage = pkgs.lib.callPackageWith (pkgs);
-    marge = callPackage ./marge.nix {};
-    version = marge.version;
+let
+  marge = pkgs.callPackage ./marge.nix {};
+  version = marge.version;
+  basicShadow =
+    # minimal user setup, so ssh won't whine 'No user exists for uid 0'
+    pkgs.runCommand "basic-shadow-setup" {}
+      ''
+        mkdir -p $out
+        cd $out
+        ${pkgs.dockerTools.shadowSetup}
+        mkdir -p root/.ssh
+      '';
 in
-pkgs.dockerTools.buildImage {
-  name = "smarkets/marge-bot";
-  tag = "${version}";
-  # minimal user setup, so ssh won't whine 'No user exists for uid 0'
-  runAsRoot = ''
-  #!${pkgs.stdenv.shell}
-  ${pkgs.dockerTools.shadowSetup}
-  mkdir -p /root/.ssh
-  '';
-  contents = [marge pkgs.bash pkgs.coreutils pkgs.openssh pkgs.glibcLocales];
-  config = {
-    Entrypoint = [ "/bin/marge.app" ];
-    Env = ["LANG=en_US.UTF-8" ''LOCALE_ARCHIVE=/lib/locale/locale-archive''];
-  };
-}
+  pkgs.dockerTools.buildImage {
+    name = "smarkets/marge-bot";
+    tag = "${version}";
+    contents =
+      with pkgs; [
+        basicShadow
+        bash
+        coreutils
+        git
+        glibcLocales
+        openssh
+      ] ++ [ marge ];
+    config = {
+      Entrypoint = [ "/bin/marge.app" ];
+      Env = ["LANG=en_US.UTF-8" ''LOCALE_ARCHIVE=/lib/locale/locale-archive''];
+    };
+  }

marge.nix

@@ -1,22 +1,40 @@
-{pkgs ? import ./pinnedNixpkgs.nix }:
-let version = builtins.replaceStrings ["\n"] [""] (builtins.readFile ./version);
-    python = (import ./requirements.nix { inherit pkgs; });
-    py = python.packages;
+{ pkgs
+, lib
+}:
+let
+  python = import ./requirements.nix { inherit pkgs; };
+  version = lib.fileContents ./version;
 in
-python.mkDerivation {
-  version = "${version}";
-  name = "marge-${version}";
-  src = ./.;
-  buildInputs = [py.pytest py.pytest-cov py.pytest-flake8 py.pytest-pylint py.pytest-runner];
-  propagatedBuildInputs = [py.ConfigArgParse py.maya py.PyYAML py.requests pkgs.openssh pkgs.git];
-  meta = {
-    homepage = "https://github.com/smarkets/marge-bot";
-    description = "A build bot for GitLab";
-    license = with pkgs.lib.licenses; [bsd3] ;
-    maintainers =  [
-      "Alexander Schmolck <[email protected]>"
-      "Jaime Lennox <[email protected]>"
+  python.mkDerivation {
+    version = "${version}";
+    name = "marge-${version}";
+    src = lib.sourceByRegex ./. [
+      "marge(/.*\.py)?"
+      "tests(/.*\.py)?"
+      "marge\.app"
+      "pylintrc"
+      "setup\.cfg"
+      "setup\.py"
+      "version"
     ];
-    platforms = pkgs.lib.platforms.linux ++ pkgs.lib.platforms.darwin;
-  };
- }
+    checkInputs = with python.packages; [
+      pytest
+      pytest-cov
+      pytest-flake8
+      pytest-pylint
+      pytest-runner
+    ];
+    propagatedBuildInputs = with python.packages; [
+      ConfigArgParse maya PyYAML requests
+    ];
+    meta = {
+      homepage = "https://github.com/smarkets/marge-bot";
+      description = "A build bot for GitLab";
+      license = lib.licenses.bsd3;
+      maintainers =  [
+        "Alexander Schmolck <[email protected]>"
+        "Jaime Lennox <[email protected]>"
+      ];
+      platforms = pkgs.lib.platforms.linux ++ pkgs.lib.platforms.darwin;
+    };
+  }

marge/bot.py

@@ -14,7 +14,7 @@
 MergeRequest = merge_request_module.MergeRequest
 
 
-class Bot(object):
+class Bot:
     def __init__(self, *, api, config):
         self._api = api
         self._config = config

marge/gitlab.py

@@ -5,7 +5,7 @@
 import requests
 
 
-class Api(object):
+class Api:
     def __init__(self, gitlab_url, auth_token):
         self._auth_token = auth_token
         self._api_base_url = gitlab_url.rstrip('/') + '/api/v4'
@@ -191,7 +191,7 @@ class UnexpectedError(ApiError):
     pass
 
 
-class Resource(object):
+class Resource:
     def __init__(self, api, info):
         self._info = info
         self._api = api

marge/interval.py

@@ -30,7 +30,7 @@ def find_weekday(string_or_day):
     raise ValueError('Not a week day: %r' % string_or_day)
 
 
-class WeeklyInterval(object):
+class WeeklyInterval:
     def __init__(self, from_weekday, from_time, to_weekday, to_time):
         from_weekday = find_weekday(from_weekday)
         to_weekday = find_weekday(to_weekday)
@@ -111,7 +111,7 @@ def _interval_covers(self, date):
         return True
 
 
-class IntervalUnion(object):
+class IntervalUnion:
     def __init__(self, iterable):
         self._intervals = list(iterable)
 

marge/job.py

@@ -14,7 +14,7 @@
 from .pipeline import Pipeline
 
 
-class MergeJob(object):
+class MergeJob:
 
     def __init__(self, *, api, user, project, repo, options):
         self._api = api
@@ -59,8 +59,7 @@ def ensure_mergeable_mr(self, merge_request):
         if state not in ('opened', 'reopened', 'locked'):
             if state in ('merged', 'closed'):
                 raise SkipMerge('The merge request is already {}!'.format(state))
-            else:
-                raise CannotMerge('The merge request is in an unknown state: {}'.format(state))
+            raise CannotMerge('The merge request is in an unknown state: {}'.format(state))
 
         if self.during_merge_embargo():
             raise SkipMerge('Merge embargo!')

marge/pylintrc

@@ -0,0 +1 @@
+../pylintrc

marge/single_merge_job.py

@@ -115,14 +115,14 @@ def update_merge_request_and_accept(self, approvals):
                     raise CannotMerge(
                         'The request was marked as WIP as I was processing it (maybe a WIP commit?)'
                     )
-                elif merge_request.state == 'reopened':
+                if merge_request.state == 'reopened':
                     raise CannotMerge(
                         'GitLab refused to merge this branch. I suspect that a Push Rule or a git-hook '
                         'is rejecting my commits; maybe my email needs to be white-listed?'
                     )
-                elif merge_request.state == 'closed':
+                if merge_request.state == 'closed':
                     raise CannotMerge('Someone closed the merge request while I was attempting to merge it.')
-                elif merge_request.state == 'merged':
+                if merge_request.state == 'merged':
                     # We are not covering any observed behaviour here, but if at this
                     # point the request is merged, our job is done, so no need to complain
                     log.info('Merge request is already merged, someone was faster!')

marge/store.py

@@ -3,7 +3,7 @@
 from . import git
 
 
-class RepoManager(object):
+class RepoManager:
 
     def __init__(self, user, root_dir, ssh_key_file=None, timeout=None, reference=None):
         self._root_dir = root_dir

nix/sources.json

@@ -0,0 +1,25 @@
+{
+    "nixpkgs": {
+        "url": "https://github.com/NixOS/nixpkgs-channels/archive/915ce0f1e1a75adec7079ddb6cd3ffba5036b3fc.tar.gz",
+        "owner": "NixOS",
+        "branch": "nixos-19.03",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz",
+        "repo": "nixpkgs-channels",
+        "type": "tarball",
+        "sha256": "1kmx29i3xy4701z4lgmv5xxslb1djahrjxmrf83ig1whb4vgk4wm",
+        "description": "Nixpkgs/NixOS branches that track the Nixpkgs/NixOS channels",
+        "rev": "915ce0f1e1a75adec7079ddb6cd3ffba5036b3fc"
+    },
+    "niv": {
+        "homepage": "https://github.com/nmattia/niv",
+        "url": "https://github.com/nmattia/niv/archive/e5e441998ede88dfce5b8b9a7ea99e1e0f1102fa.tar.gz",
+        "owner": "nmattia",
+        "branch": "master",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz",
+        "repo": "niv",
+        "type": "tarball",
+        "sha256": "0s3pwakbp9qmwzznl8xd3smmymz1s2vrvyip8yizqdllaps4pf18",
+        "description": "Easy dependency management for Nix projects",
+        "rev": "e5e441998ede88dfce5b8b9a7ea99e1e0f1102fa"
+    }
+}

nix/sources.nix

@@ -0,0 +1,11 @@
+# Read in the json spec for packages we want (so it can be auto-updated).
+# niv: no_update
+
+# make travis happy, reasonably new nix doesn't need this
+let mapAttrs = builtins.mapAttrs or
+    (f: set:
+      builtins.listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (builtins.attrNames set)));
+in with builtins;
+   mapAttrs
+     (_: spec: spec // { outPath = fetchTarball { inherit (spec) url sha256; }; })
+     (fromJSON (readFile ./sources.json))

pinnedNixpkgs.nix

@@ -1,9 +1,8 @@
 let
-  fetchFromGitHub = (import <nixpkgs> {}).fetchFromGitHub;
-  pkgs = import (fetchFromGitHub {
-                   owner  = "NixOS";
-                   repo   = "nixpkgs";
-                   rev    = "90afb0c10fe6f437fca498298747b2bcb6a77d39";
-                   sha256 = "0mvzdw5aygi1vjnvm0bc8bp7iwb9rypiqg749m6a6km84m7srm0w";
-                 }) {};
-in pkgs
+  spec = builtins.fromJSON (builtins.readFile ./pinnedNixpkgs.src.json);
+  src = builtins.fetchTarball {
+    url = "https://github.com/${spec.owner}/${spec.repo}/archive/${spec.rev}.tar.gz";
+    sha256 = spec.sha256;
+  };
+in
+  import src {}

pinnedNixpkgs.src.json

@@ -0,0 +1,6 @@
+{
+  "owner": "NixOS",
+  "repo": "nixpkgs",
+  "rev": "1985e76c2ee8b0bfc144ff4a495d68bb432d9153",
+  "sha256": "0lvdd749idxkqb91damn504zd310d9hqxhph8mg874wpld5kv87n"
+}

pylintrc

@@ -14,6 +14,7 @@ disable=bad-continuation,
         fixme,
         missing-docstring,
         no-self-use,
+        unsubscriptable-object
 
 [SIMILARITIES]
 min-similarity-lines=10