Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Copy Files From External Storage To Local Storage for Import #149

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open

Copy Files From External Storage To Local Storage for Import #149

Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1363232
Copy Files From External Storage To Local Storage for Import
wattnpapa Sep 3, 2024
cbf2298
Merge branch 'main' into loadLocalTemp
wattnpapa Dec 30, 2024
9bba682
php 8.4
wattnpapa Dec 30, 2024
34b6a76
Optimize code / add tests
wattnpapa Dec 30, 2024
936561f
Add return value ressource
wattnpapa Dec 31, 2024
50c0b5c
Use mixed
wattnpapa Dec 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 62 additions & 32 deletions src/Snapshot.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,15 @@
namespace Spatie\DbSnapshots;

use Carbon\Carbon;
use Illuminate\Contracts\Filesystem\Factory;
use Illuminate\Filesystem\FilesystemAdapter as Disk;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\LazyCollection;
use Spatie\DbSnapshots\Events\DeletedSnapshot;
use Spatie\DbSnapshots\Events\DeletingSnapshot;
use Spatie\DbSnapshots\Events\LoadedSnapshot;
use Spatie\DbSnapshots\Events\LoadingSnapshot;
use Spatie\TemporaryDirectory\TemporaryDirectory;

class Snapshot
{
Expand All @@ -25,6 +27,8 @@ class Snapshot

public const STREAM_BUFFER_SIZE = 16384;

protected Factory $filesystemFactory;

public function __construct(Disk $disk, string $fileName)
{
$this->disk = $disk;
Expand All @@ -39,6 +43,8 @@ public function __construct(Disk $disk, string $fileName)
}

$this->name = pathinfo($fileName, PATHINFO_FILENAME);

$this->filesystemFactory = app(Factory::class);
}

public function useStream(): self
Expand Down Expand Up @@ -90,45 +96,69 @@ protected function shouldIgnoreLine(string $line): bool

protected function loadStream(?string $connectionName = null): void
{
LazyCollection::make(function () {
$stream = $this->compressionExtension === 'gz'
? gzopen($this->disk->path($this->fileName), 'r')
: $this->disk->readStream($this->fileName);

$statement = '';
while (! feof($stream)) {
$chunk = $this->compressionExtension === 'gz'
$directory = (new TemporaryDirectory(config('db-snapshots.temporary_directory_path')))->create();

config([
'filesystems.disks.' . self::class => [
'driver' => 'local',
'root' => $directory->path(),
'throw' => false,
]
]);

$localDisk = $this->filesystemFactory->disk(self::class);

try {
LazyCollection::make(function () use ($localDisk) {
$localDisk->writeStream($this->fileName, $this->disk->readStream($this->fileName));
wattnpapa marked this conversation as resolved.
Show resolved Hide resolved

$stream = $this->compressionExtension === 'gz'
? gzopen($localDisk->path($this->fileName), 'r')
: $localDisk->readStream($this->fileName);

$statement = '';
while (!feof($stream)) {
$chunk = $this->compressionExtension === 'gz'
? gzread($stream, self::STREAM_BUFFER_SIZE)
: fread($stream, self::STREAM_BUFFER_SIZE);

$lines = explode("\n", $chunk);
foreach ($lines as $idx => $line) {
if ($this->shouldIgnoreLine($line)) {
continue;
$lines = explode("\n", $chunk);
foreach ($lines as $idx => $line) {
if ($this->shouldIgnoreLine($line)) {
continue;
}

$statement .= $line;

// Carry-over the last line to the next chunk since it
// is possible that this chunk finished mid-line right on
// a semi-colon.
if (count($lines) == $idx + 1) {
break;
}

if (str_ends_with(trim($statement), ';')) {
yield $statement;
$statement = '';
}
}
}

$statement .= $line;

// Carry-over the last line to the next chunk since it
// is possible that this chunk finished mid-line right on
// a semi-colon.
if (count($lines) == $idx + 1) {
break;
}
if (str_ends_with(trim($statement), ';')) {
yield $statement;
}

if (str_ends_with(trim($statement), ';')) {
yield $statement;
$statement = '';
}
if ($this->compressionExtension === 'gz') {
gzclose($stream);
} else {
wattnpapa marked this conversation as resolved.
Show resolved Hide resolved
fclose($stream);
}
}

if (str_ends_with(trim($statement), ';')) {
yield $statement;
}
})->each(function (string $statement) use ($connectionName) {
DB::connection($connectionName)->unprepared($statement);
});
})->each(function (string $statement) use ($connectionName) {
DB::connection($connectionName)->unprepared($statement);
});
} finally {
$directory->delete();
}
}

public function delete(): void
Expand Down
Loading