Mirroring repositories locally to deployer

.php_cs

@@ -5,7 +5,6 @@ $finder = Symfony\Component\Finder\Finder::create()
     ->in('app')
     ->in('database')
     ->name('*.php')
-    ->notName('Kernel.php') // There has to be a better way to do this!
     ->ignoreDotFiles(true)
     ->ignoreVCS(true);
 
@@ -70,4 +69,4 @@ return Symfony\CS\Config\Config::create()
     ->level(Symfony\CS\FixerInterface::PSR2_LEVEL)
     ->fixers($fixers)
     ->finder($finder)
-    ->setUsingCache(true);
+    ->setUsingCache(true);

.phpci.yml

@@ -10,6 +10,7 @@ build_settings:
         - "storage"
         - "tests"
         - "vendor"
+        - "create-release"
 
 setup:
     wipe:

Vagrantfile

@@ -99,6 +99,8 @@ Vagrant.configure("2") do |config|
     config.vm.provision "shell", inline: "echo 'alias phpunit=\"php $(which phpunit)\"' >> /home/vagrant/.profile"
     config.vm.provision "shell", inline: "echo 'export PATH=/var/www/deployer/vendor/bin:$PATH' >> /home/vagrant/.profile"
 
+    config.vm.provision "shell", inline: "echo 'export PATH=/var/www/deployer/vendor/bin:$PATH' >> /home/vagrant/.profile"
+
     # Update composer on each boot
     config.vm.provision "shell", inline: "sudo /usr/local/bin/composer self-update", run: "always"
 end

app/Console/Commands/ClearOldKeys.php

@@ -41,8 +41,8 @@ public function __construct()
     public function handle()
     {
         // Clear out old SSH key files
-        $files   = glob(storage_path() . '/app/*ssh*'); // sshkey and gitssh
-        $folders = glob(storage_path() . '/app/*clone*'); // cloned copies of code
+        $files   = glob(storage_path('app/') . '*ssh*'); // sshkey and gitssh
+        $folders = glob(storage_path('app/') . '*clone*'); // cloned copies of code
 
         $this->info('Found ' . count($files) . ' files and ' . count($folders) . ' folders to purge');
 

app/Console/Commands/ClearOrphanMirrors.php

@@ -0,0 +1,75 @@
+<?php
+
+namespace REBELinBLUE\Deployer\Console\Commands;
+
+use Illuminate\Console\Command;
+use REBELinBLUE\Deployer\Project;
+use Symfony\Component\Process\Process;
+
+/**
+ * Checks for and cleans up orphaned git mirrors.
+ */
+class ClearOrphanMirrors extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'deployer:purge-mirrors';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Purges git mirrors which are no longer in use by projects';
+
+    /**
+     * Create a new command instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        parent::__construct();
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return mixed
+     */
+    public function handle()
+    {
+        $current_mirrors = [];
+
+        Project::where('is_template', false)->chunk(10, function ($projects) use (&$current_mirrors) {
+            foreach ($projects as $project) {
+                $current_mirrors[] = $project->mirrorPath();
+            }
+        });
+
+        $current_mirrors = collect($current_mirrors);
+
+        $all_mirrors = collect(glob(storage_path('app/mirrors/') . '*.git'));
+
+        // Compare the 2 collections get a list of mirrors which are no longer in use
+        $orphan_mirrors = $all_mirrors->diff($current_mirrors);
+
+        $this->info('Found ' . $orphan_mirrors->count() . ' orphaned mirrors');
+
+        // Now loop through the mirrors and delete them from storage
+        foreach ($orphan_mirrors as $mirror_dir) {
+            $process = new Process("rm -rf {$mirror_dir}");
+            $process->setTimeout(null);
+            $process->run();
+
+            if ($process->isSuccessful()) {
+                $this->info('Deleted ' . basename($mirror_dir));
+            } else {
+                $this->info('Failed to delete ' . basename($mirror_dir));
+            }
+        }
+    }
+}

app/Console/Commands/CreateUser.php

@@ -84,7 +84,8 @@ public function handle()
 
             event(new UserWasCreated($user, $arguments['password']));
         } elseif ($password_generated) {
-            $message .= ', however you elected to not email the account details to them. Their password is ' . $arguments['password'];
+            $message .= ', however you elected to not email the account details to them. ';
+            $message .= 'Their password is ' . $arguments['password'];
         }
 
         $this->info($message);

app/Console/Commands/InstallApp.php

@@ -118,9 +118,9 @@ public function handle()
         $this->line('');
         $this->comment('2. Setup the cronjobs, see "crontab"');
         $this->line('');
-        $this->comment('3. Setup the socket server and queue runner, see "supervisor.conf" for an example of how to do this with supervisor');
+        $this->comment('3. Setup the socket server & queue runner, see "supervisor.conf" for an example commands');
         $this->line('');
-        $this->comment('4. Ensure that "storage" and "public/upload" are writable by the webserver, for example run "chmod -R 777 storage"');
+        $this->comment('4. Ensure that "storage" and "public/upload" are writable by the webserver');
         $this->line('');
         $this->comment('5. (Optional) Setup logrotate, see "logrotate.conf"');
         $this->line('');
@@ -391,7 +391,8 @@ private function getInstallInformation()
         if (count($locales) === 1) {
             $locale = $locales[0];
         } else {
-            $locale = $this->choice('Language', $locales, array_search(Config::get('app.fallback_locale'), $locales, true));
+            $default = array_search(Config::get('app.fallback_locale'), $locales, true);
+            $locale  = $this->choice('Language', $locales, $default);
         }
 
         return [
@@ -584,7 +585,9 @@ protected function checkRequirements()
         }
 
         if (!count($this->getDatabaseDrivers())) {
-            $this->error('At least 1 PDO database driver is required. Either sqlite, mysql, pgsql or sqlsrv, check your php.ini file');
+            $this->error(
+                'At least 1 PDO driver is required. Either sqlite, mysql, pgsql or sqlsrv, check your php.ini file'
+            );
             $errors = true;
         }
 
@@ -599,7 +602,7 @@ protected function checkRequirements()
         }
 
         // Programs needed in $PATH
-        $required_commands = ['ssh', 'ssh-keygen', 'git', 'scp'];
+        $required_commands = ['ssh', 'ssh-keygen', 'git', 'scp', 'tar', 'gzip'];
 
         foreach ($required_commands as $command) {
             $process = new Process('which ' . $command);
@@ -630,7 +633,7 @@ protected function checkRequirements()
 
         if ($errors) {
             $this->line('');
-            $this->block('Deployer cannot be installed, as not all requirements are met. Please review the errors above before continuing.');
+            $this->block('Deployer cannot be installed. Please review the errors above before continuing.');
             $this->line('');
 
             return false;

app/Console/Commands/UpdateApp.php

@@ -43,7 +43,10 @@ public function __construct()
      */
     public function handle()
     {
-        if (!$this->verifyInstalled() || $this->hasRunningDeployments() || $this->composerOutdated() || !$this->checkRequirements()) {
+        if (!$this->verifyInstalled() ||
+            $this->hasRunningDeployments() ||
+            $this->composerOutdated() ||
+            !$this->checkRequirements()) {
             return;
         }
 

app/Console/Commands/UpdateGitMirrors.php

@@ -0,0 +1,61 @@
+<?php
+
+namespace REBELinBLUE\Deployer\Console\Commands;
+
+use Carbon\Carbon;
+use Illuminate\Console\Command;
+use Illuminate\Foundation\Bus\DispatchesJobs;
+use REBELinBLUE\Deployer\Jobs\UpdateGitMirror;
+use REBELinBLUE\Deployer\Project;
+
+/**
+ * Updates the mirrors for all git repositories.
+ */
+class UpdateGitMirrors extends Command
+{
+    use DispatchesJobs;
+
+    const UPDATES_TO_QUEUE         = 3;
+    const UPDATE_FREQUENCY_MINUTES = 5;
+
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'deployer:update-mirrors';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Pulls in updates for git mirrors';
+
+    /**
+     * Create a new command instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        parent::__construct();
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return mixed
+     */
+    public function handle()
+    {
+        $last_mirrored_since = Carbon::now()->subMinutes(self::UPDATE_FREQUENCY_MINUTES);
+        $todo                = self::UPDATES_TO_QUEUE;
+
+        Project::where('last_mirrored', '<', $last_mirrored_since)->chunk($todo, function ($projects) {
+            foreach ($projects as $project) {
+                $this->dispatch(new UpdateGitMirror($project));
+            }
+        });
+    }
+}

app/Console/Kernel.php

@@ -2,9 +2,10 @@
 
 namespace REBELinBLUE\Deployer\Console;
 
-use REBELinBLUE\Deployer\Bootstrap\ConfigureLogging;
 use Illuminate\Console\Scheduling\Schedule;
 use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
+use REBELinBLUE\Deployer\Bootstrap\ConfigureLogging;
+use REBELinBLUE\Deployer\Bootstrap\ConfigureLogging as ConsoleLogging;
 
 /**
  * Kernel class.
@@ -16,7 +17,7 @@ class Kernel extends ConsoleKernel
      * @var array
      */
     protected $customBooters = [
-        \Illuminate\Foundation\Bootstrap\ConfigureLogging::class => ConfigureLogging::class,
+        \Illuminate\Foundation\Bootstrap\ConfigureLogging::class => ConsoleLogging::class,
     ];
 
     /**
@@ -37,8 +38,10 @@ class Kernel extends ConsoleKernel
         \REBELinBLUE\Deployer\Console\Commands\CheckUrl::class,
         \REBELinBLUE\Deployer\Console\Commands\CreateUser::class,
         \REBELinBLUE\Deployer\Console\Commands\ClearOrphanAvatars::class,
+        \REBELinBLUE\Deployer\Console\Commands\ClearOrphanMirrors::class,
         \REBELinBLUE\Deployer\Console\Commands\ClearStalledDeployment::class,
         \REBELinBLUE\Deployer\Console\Commands\ClearOldKeys::class,
+        \REBELinBLUE\Deployer\Console\Commands\UpdateGitMirrors::class,
         \REBELinBLUE\Deployer\Console\Commands\InstallApp::class,
         \REBELinBLUE\Deployer\Console\Commands\UpdateApp::class,
         \REBELinBLUE\Deployer\Console\Commands\ResetApp::class,
@@ -57,6 +60,10 @@ protected function schedule(Schedule $schedule)
                  ->everyFiveMinutes()
                  ->withoutOverlapping();
 
+        $schedule->command('deployer:update-mirrors')
+                 ->everyFiveMinutes()
+                 ->withoutOverlapping();
+
         $schedule->command('deployer:checkurls')
                  ->everyFiveMinutes()
                  ->withoutOverlapping();
@@ -67,6 +74,10 @@ protected function schedule(Schedule $schedule)
                  ->at('00:30')
                  ->withoutOverlapping();
 
+        $schedule->command('deployer:purge-mirrors')
+                 ->daily()
+                 ->withoutOverlapping();
+
         $schedule->command('deployer:purge-temp')
                  ->hourly()
                  ->withoutOverlapping();

app/Deployment.php

@@ -3,6 +3,7 @@
 namespace REBELinBLUE\Deployer;
 
 use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\Relations\HasMany;
 use Illuminate\Database\Eloquent\SoftDeletes;
 use Illuminate\Support\Facades\App;
 use Illuminate\Support\Facades\DB;
@@ -35,14 +36,15 @@ class Deployment extends Model implements PresentableInterface, RuntimeInterface
      *
      * @var array
      */
-    protected $fillable = ['reason', 'branch', 'project_id', 'source', 'build_url'];
+    protected $fillable = ['reason', 'branch', 'project_id', 'source', 'build_url',
+                           'commit', 'committer_email', 'committer', ];
 
     /**
      * The attributes excluded from the model's JSON form.
      *
      * @var array
      */
-    protected $hidden = ['created_at', 'deleted_at', 'updated_at', 'user'];
+    protected $hidden = ['created_at', 'deleted_at', 'updated_at', 'user', 'commands'];
 
     /**
      * Additional attributes to include in the JSON representation.
@@ -107,6 +109,40 @@ public function user()
                     ->withTrashed();
     }
 
+    /**
+     * Define a command attribute to be able to access to commands relationship.
+     *
+     * @return Command
+     */
+    public function getCommandsAttribute()
+    {
+        if (!$this->relationLoaded('commands')) {
+            $this->loadCommands();
+        }
+
+        return $this->getRelation('commands');
+    }
+
+    /**
+     * Query the DB and load the HasMany relationship for commands.
+     *
+     * @return Deployment
+     */
+    private function loadCommands()
+    {
+        $collection = Command::join('deploy_steps', 'commands.id', '=', 'deploy_steps.command_id')
+                             ->where('deploy_steps.deployment_id', $this->getKey())
+                             ->distinct()
+                             ->orderBy('step')
+                             ->orderBy('order')
+                             ->get(['commands.*', 'deployment_id']);
+
+        $hasMany = new HasMany(Command::query(), $this, 'deployment_id', 'id');
+        $hasMany->matchMany([$this], $collection, 'commands');
+
+        return $this;
+    }
+
     /**
      * Has many relationship.
      *

app/Http/Controllers/Admin/UserController.php

@@ -33,7 +33,7 @@ public function index()
     {
         return view('admin.users.listing', [
             'title' => Lang::get('users.manage'),
-            'users' => $this->repository->getAll()->toJson(), // Because PresentableInterface toJson() is not working in the view
+            'users' => $this->repository->getAll()->toJson(), // PresentableInterface toJson() is not working in view
         ]);
     }