Skip to content

Commit

Permalink
Add pack-refs command to the CLI
Browse files Browse the repository at this point in the history
This command can be used to optimize storage of references.

For a RefDirectory database, it packs non-symbolic, loose refs into
packed-refs. By default, only the references under '$GIT_DIR/refs/tags'
are packed. The '--all' option can be used to pack all the references
under '$GIT_DIR/refs'.

For Reftable, all refs are compacted into a single table.

Change-Id: I92e786403f8638d35ae3037844a7ad89e8959e02
  • Loading branch information
yash-c8i committed Nov 22, 2024
1 parent f295477 commit 6fa28d7
Show file tree
Hide file tree
Showing 14 changed files with 317 additions and 54 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
/*
* Copyright (c) 2024 Qualcomm Innovation Center, Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Distribution License v. 1.0 which is available at
* https://www.eclipse.org/org/documents/edl-v10.php.
*
* SPDX-License-Identifier: BSD-3-Clause
*/

package org.eclipse.jgit.pgm;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;

import java.io.File;

import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.internal.storage.file.FileRepository;
import org.eclipse.jgit.lib.CLIRepositoryTestCase;
import org.eclipse.jgit.lib.ConfigConstants;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.Ref;
import org.junit.Before;
import org.junit.Test;

public class PackRefsTest extends CLIRepositoryTestCase {
private Git git;

@Override
@Before
public void setUp() throws Exception {
super.setUp();
git = new Git(db);
git.commit().setMessage("initial commit").call();
}

@Test
public void tagPacked() throws Exception {
git.tag().setName("test").call();
git.packRefs().call();
assertEquals(Ref.Storage.PACKED,
git.getRepository().exactRef("refs/tags/test").getStorage());
}

@Test
public void nonTagRefNotPackedWithoutAll() throws Exception {
git.branchCreate().setName("test").call();
git.packRefs().call();
assertEquals(Ref.Storage.LOOSE,
git.getRepository().exactRef("refs/heads/test").getStorage());
}

@Test
public void nonTagRefPackedWithAll() throws Exception {
git.branchCreate().setName("test").call();
git.packRefs().setAll(true).call();
assertEquals(Ref.Storage.PACKED,
git.getRepository().exactRef("refs/heads/test").getStorage());
}

@Test
public void refTableCompacted() throws Exception {
((FileRepository) git.getRepository()).convertRefStorage(
ConfigConstants.CONFIG_REF_STORAGE_REFTABLE, false, false);

git.commit().setMessage("test commit").call();
File tableDir = new File(db.getDirectory(), Constants.REFTABLE);
File[] reftables = tableDir.listFiles();
assertNotNull(reftables);
assertTrue(reftables.length > 2);

git.packRefs().call();

reftables = tableDir.listFiles();
assertNotNull(reftables);
assertEquals(2, reftables.length);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ org.eclipse.jgit.pgm.LsTree
org.eclipse.jgit.pgm.Merge
org.eclipse.jgit.pgm.MergeBase
org.eclipse.jgit.pgm.MergeTool
org.eclipse.jgit.pgm.PackRefs
org.eclipse.jgit.pgm.Push
org.eclipse.jgit.pgm.ReceivePack
org.eclipse.jgit.pgm.Reflog
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,7 @@ updating=Updating {0}..{1}
usage_Abbrev=Instead of using the default number of hexadecimal digits (which will vary according to the number of objects in the repository with a default of 7) of the abbreviated object name, use <n> digits, or as many digits as needed to form a unique object name. An <n> of 0 will suppress long format, only showing the closest tag.
usage_addRenormalize=Apply the "clean" process freshly to tracked files to forcibly add them again to the index. This implies -u.
usage_Aggressive=This option will cause gc to more aggressively optimize the repository at the expense of taking much more time
usage_All=Pack all refs, except hidden refs, broken refs, and symbolic refs.
usage_AlwaysFallback=Show uniquely abbreviated commit object as fallback
usage_bareClone=Make a bare Git repository. That is, instead of creating [DIRECTORY] and placing the administrative files in [DIRECTORY]/.git, make the [DIRECTORY] itself the $GIT_DIR.
usage_extraArgument=Pass an extra argument to a merge driver. Currently supported are "-X ours" and "-X theirs".
Expand Down Expand Up @@ -300,6 +301,7 @@ usage_Match=Only consider tags matching the given glob(7) pattern or patterns, e
usage_MergeBase=Find as good common ancestors as possible for a merge
usage_MergesTwoDevelopmentHistories=Merges two development histories
usage_PackKeptObjects=Include objects in packs locked by a ".keep" file when repacking
usage_PackRefs=Pack heads and tags for efficient repository access
usage_PreserveOldPacks=Preserve old pack files by moving them into the preserved subdirectory instead of deleting them after repacking
usage_PrunePreserved=Remove the preserved subdirectory containing previously preserved old pack files before repacking, and before preserving more old pack files
usage_ReadDirCache= Read the DirCache 100 times
Expand Down
34 changes: 34 additions & 0 deletions org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/PackRefs.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
* Copyright (c) 2024 Qualcomm Innovation Center, Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Distribution License v. 1.0 which is available at
* https://www.eclipse.org/org/documents/edl-v10.php.
*
* SPDX-License-Identifier: BSD-3-Clause
*/

package org.eclipse.jgit.pgm;

import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.TextProgressMonitor;
import org.kohsuke.args4j.Option;

@Command(common = true, usage = "usage_PackRefs")
class PackRefs extends TextBuiltin {
@Option(name = "--all", usage = "usage_All")
private boolean all;

@Override
protected void run() {
Git git = Git.wrap(db);
try {
git.packRefs().setProgressMonitor(new TextProgressMonitor(errw))
.setAll(all).call();
} catch (GitAPIException e) {
throw die(e.getMessage(), e);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
import static org.junit.Assert.assertSame;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.concurrent.BrokenBarrierException;
Expand All @@ -31,6 +30,8 @@
import java.util.concurrent.TimeUnit;

import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.PackRefsCommand;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.junit.TestRepository.BranchBuilder;
import org.eclipse.jgit.lib.ConfigConstants;
import org.eclipse.jgit.lib.Constants;
Expand All @@ -49,7 +50,7 @@ public void looseRefPacked() throws Exception {
RevBlob a = tr.blob("a");
tr.lightweightTag("t", a);

gc.packRefs();
packRefs(false);
assertSame(repo.exactRef("refs/tags/t").getStorage(), Storage.PACKED);
}

Expand All @@ -60,7 +61,7 @@ public void emptyRefDirectoryDeleted() throws Exception {
String name = repo.findRef(ref).getName();
Path dir = repo.getCommonDirectory().toPath().resolve(name).getParent();
assertNotNull(dir);
gc.packRefs();
packRefs(true);
assertFalse(Files.exists(dir));
}

Expand All @@ -75,9 +76,9 @@ public void concurrentOnlyOneWritesPackedRefs() throws Exception {
Callable<Integer> packRefs = () -> {
syncPoint.await();
try {
gc.packRefs();
packRefs(false);
return 0;
} catch (IOException e) {
} catch (GitAPIException e) {
return 1;
}
};
Expand All @@ -102,7 +103,7 @@ public void whileRefLockedRefNotPackedNoError()
"refs/tags/t1"));
try {
refLock.lock();
gc.packRefs();
packRefs(false);
} finally {
refLock.unlock();
}
Expand Down Expand Up @@ -145,7 +146,7 @@ public boolean isForceUpdate() {

Future<Result> result2 = pool.submit(() -> {
refUpdateLockedRef.await();
gc.packRefs();
packRefs(false);
packRefsDone.await();
return null;
});
Expand Down Expand Up @@ -173,19 +174,20 @@ public void dontPackHEAD_nonBare() throws Exception {
assertEquals(repo.exactRef("HEAD").getTarget().getName(),
"refs/heads/master");
assertNull(repo.exactRef("HEAD").getTarget().getObjectId());
gc.packRefs();
PackRefsCommand packRefsCommand = git.packRefs().setAll(true);
packRefsCommand.call();
assertSame(repo.exactRef("HEAD").getStorage(), Storage.LOOSE);
assertEquals(repo.exactRef("HEAD").getTarget().getName(),
"refs/heads/master");
assertNull(repo.exactRef("HEAD").getTarget().getObjectId());

git.checkout().setName("refs/heads/side").call();
gc.packRefs();
packRefsCommand.call();
assertSame(repo.exactRef("HEAD").getStorage(), Storage.LOOSE);

// check for detached HEAD
git.checkout().setName(first.getName()).call();
gc.packRefs();
packRefsCommand.call();
assertSame(repo.exactRef("HEAD").getStorage(), Storage.LOOSE);
}

Expand All @@ -208,17 +210,22 @@ public void dontPackHEAD_bare() throws Exception {
assertEquals(repo.exactRef("HEAD").getTarget().getName(),
"refs/heads/master");
assertNull(repo.exactRef("HEAD").getTarget().getObjectId());
gc.packRefs();
packRefs(true);
assertSame(repo.exactRef("HEAD").getStorage(), Storage.LOOSE);
assertEquals(repo.exactRef("HEAD").getTarget().getName(),
"refs/heads/master");
assertNull(repo.exactRef("HEAD").getTarget().getObjectId());

// check for non-detached HEAD
repo.updateRef(Constants.HEAD).link("refs/heads/side");
gc.packRefs();
packRefs(true);
assertSame(repo.exactRef("HEAD").getStorage(), Storage.LOOSE);
assertEquals(repo.exactRef("HEAD").getTarget().getObjectId(),
second.getId());
}

private void packRefs(boolean all) throws GitAPIException {
new PackRefsCommand(repo).setAll(all).call();
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -597,6 +597,8 @@ packInaccessible=Failed to access pack file {0}, caught {1} consecutive errors w
packingCancelledDuringObjectsWriting=Packing cancelled during objects writing
packObjectCountMismatch=Pack object count mismatch: pack {0} index {1}: {2}
packRefs=Pack refs
packRefsFailed=Packing refs failed
packRefsSuccessful=Packed refs successfully
packSizeNotSetYet=Pack size not yet set since it has not yet been received
packTooLargeForIndexVersion1=Pack too large for index version 1
packWasDeleted=Pack file {0} was deleted, removing it from pack list
Expand Down
10 changes: 10 additions & 0 deletions org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java
Original file line number Diff line number Diff line change
Expand Up @@ -713,6 +713,16 @@ public GarbageCollectCommand gc() {
return new GarbageCollectCommand(repo);
}

/**
* Return a command object to execute a {@code PackRefs} command
*
* @return a {@link org.eclipse.jgit.api.PackRefsCommand}
* @since 7.1
*/
public PackRefsCommand packRefs() {
return new PackRefsCommand(repo);
}

/**
* Return a command object to find human-readable names of revisions.
*
Expand Down
87 changes: 87 additions & 0 deletions org.eclipse.jgit/src/org/eclipse/jgit/api/PackRefsCommand.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
/*
* Copyright (c) 2024 Qualcomm Innovation Center, Inc.
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Distribution License v. 1.0 which is available at
* https://www.eclipse.org/org/documents/edl-v10.php.
*
* SPDX-License-Identifier: BSD-3-Clause
*/

package org.eclipse.jgit.api;

import java.io.IOException;

import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.JGitInternalException;
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.lib.NullProgressMonitor;
import org.eclipse.jgit.lib.ProgressMonitor;
import org.eclipse.jgit.lib.Repository;

/**
* Optimize storage of references.
*
* @since 7.1
*/
public class PackRefsCommand extends GitCommand<String> {
private ProgressMonitor monitor;

private boolean all;

/**
* Creates a new {@link PackRefsCommand} instance with default values.
*
* @param repo
* the repository this command will be used on
*/
public PackRefsCommand(Repository repo) {
super(repo);
this.monitor = NullProgressMonitor.INSTANCE;
}

/**
* Set progress monitor
*
* @param monitor
* a progress monitor
* @return this instance
*/
public PackRefsCommand setProgressMonitor(ProgressMonitor monitor) {
this.monitor = monitor;
return this;
}

/**
* Specify whether to pack all the references.
*
* @param all
* if <code>true</code> all the loose refs will be packed
* @return this instance
*/
public PackRefsCommand setAll(boolean all) {
this.all = all;
return this;
}

/**
* Whether to pack all the references
*
* @return whether to pack all the references
*/
public boolean isAll() {
return all;
}

@Override
public String call() throws GitAPIException {
checkCallable();
try {
repo.getRefDatabase().packRefs(monitor, this);
return JGitText.get().packRefsSuccessful;
} catch (IOException e) {
throw new JGitInternalException(JGitText.get().packRefsFailed, e);
}
}
}
2 changes: 2 additions & 0 deletions org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
Original file line number Diff line number Diff line change
Expand Up @@ -627,6 +627,8 @@ public static JGitText get() {
/***/ public String packingCancelledDuringObjectsWriting;
/***/ public String packObjectCountMismatch;
/***/ public String packRefs;
/***/ public String packRefsFailed;
/***/ public String packRefsSuccessful;
/***/ public String packSizeNotSetYet;
/***/ public String packTooLargeForIndexVersion1;
/***/ public String packWasDeleted;
Expand Down
Loading

0 comments on commit 6fa28d7

Please sign in to comment.