Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Issue 5287- Endpoints to get the md5 hash of the original model file #5328

Merged
merged 20 commits into from
Jan 21, 2025
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
9de241c
first commit - create endpoints/functions/tests
Bogdan-Crn Dec 23, 2024
f3c8bd4
eslint fix
Bogdan-Crn Dec 23, 2024
910d949
Merge branch 'staging' of github.com:3drepo/3drepo.io into ISSUE_5287
Bogdan-Crn Dec 23, 2024
0f0b898
update tests and lint adjustments
Bogdan-Crn Dec 30, 2024
246fbb9
tidy up
Bogdan-Crn Jan 2, 2025
71350c9
permission fix
Bogdan-Crn Jan 2, 2025
dabdb2a
Merge branch 'staging' of github.com:3drepo/3drepo.io into ISSUE_5287
Bogdan-Crn Jan 3, 2025
9188882
update federations test for out of order arrays
Bogdan-Crn Jan 7, 2025
2c0f5c2
Merge branch 'staging' of github.com:3drepo/3drepo.io into ISSUE_5287
Bogdan-Crn Jan 7, 2025
f16fae0
update the e2e with out of order array check
Bogdan-Crn Jan 7, 2025
d3dfe0e
lint fixes
Bogdan-Crn Jan 7, 2025
9d93e83
Update backend/src/v5/processors/teamspaces/projects/models/commons/m…
Bogdan-Crn Jan 8, 2025
874d261
federation access issue ISSUE_5287
Bogdan-Crn Jan 14, 2025
fb261ad
addressing comments #ISSUE_5287
Bogdan-Crn Jan 14, 2025
82f7d6e
Merge branch 'staging' into ISSUE_5287 #ISSUE_5287
Bogdan-Crn Jan 14, 2025
c2ef5e8
test coverage fix #ISSUE_5287
Bogdan-Crn Jan 15, 2025
9c95df6
Update backend/src/v5/services/filesManager.js
Bogdan-Crn Jan 17, 2025
3e3f9c0
address latest comments #ISSUE_5287
Bogdan-Crn Jan 20, 2025
a6c3b8b
Merge branch 'staging' of github.com:3drepo/3drepo.io into ISSUE_5287
Bogdan-Crn Jan 20, 2025
53480b5
es6 naming convention - change MD5Hash to md5Hash #ISSUE_5287
Bogdan-Crn Jan 21, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions backend/src/v5/models/fileRefs.js
Original file line number Diff line number Diff line change
Expand Up @@ -82,4 +82,10 @@ FileRefs.removeRefsByQuery = (teamspace, collection, query) => db.deleteMany(
teamspace, collectionName(collection), query,
);

FileRefs.updateRef = async (teamspace, collection, query, action) => {
await db.updateOne(
teamspace, collectionName(collection), query, action,
);
};

module.exports = FileRefs;
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,17 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

const { addModel, getContainers } = require('../../../../../models/modelSettings');
const { addModelToProject, getProjectById, removeModelFromProject } = require('../../../../../models/projectSettings');
const { getLatestRevision, getRevisionByIdOrTag } = require('../../../../../models/revisions');
const { getRefEntry, updateRef } = require('../../../../../models/fileRefs');
const { hasProjectAdminPermissions, isTeamspaceAdmin } = require('../../../../../utils/permissions/permissions');
const CryptoJs = require('crypto-js');
const { USERS_DB_NAME } = require('../../../../../models/users.constants');
const { addModel } = require('../../../../../models/modelSettings');
const UUIDParse = require('uuid-parse');
const { getFavourites } = require('../../../../../models/users');
const { getFile } = require('../../../../../services/filesManager');
const { modelTypes } = require('../../../../../models/modelSettings.constants');
const { removeModelData } = require('../../../../../utils/helper/models');

const ModelList = {};
Expand Down Expand Up @@ -50,11 +56,77 @@ ModelList.getModelList = async (teamspace, project, user, modelSettings) => {
return modelSettings.flatMap(({ _id, name, permissions: modelPerms }) => {
const perm = modelPerms ? modelPerms.find((entry) => entry.user === user) : undefined;
return (!isAdmin && !perm)
? [] : { _id,
? [] : {
_id,
name,
role: isAdmin ? USERS_DB_NAME : perm.permission,
isFavourite: favourites.includes(_id) };
isFavourite: favourites.includes(_id),
};
});
};

ModelList.getModelMD5Hash = async (teamspace, container, revision, user) => {
const [isAdmin, containers] = await Promise.all([
isTeamspaceAdmin(teamspace, user),
getContainers(teamspace, [container], { _id: 1, name: 1, permissions: 1 }),
]);
let rev;
let returnValue;

// if not allowed just return nothing
if (!isAdmin && !containers[0].permissions?.some((permission) => permission?.user === user)) return returnValue;
Bogdan-Crn marked this conversation as resolved.
Show resolved Hide resolved

// retrieve the right revision
if (revision?.length) {
Bogdan-Crn marked this conversation as resolved.
Show resolved Hide resolved
rev = await getRevisionByIdOrTag(
teamspace, container, modelTypes.CONTAINER, revision,
{ rFile: 1, timestamp: 1, fileSize: 1 },
{ includeVoid: false });
} else {
rev = await getLatestRevision(
teamspace, container, modelTypes.CONTAINER,
{ rFile: 1, timestamp: 1, fileSize: 1 });
}

// check if anything is in there
if (!rev.rFile?.length) return returnValue;
Bogdan-Crn marked this conversation as resolved.
Show resolved Hide resolved

const code = UUIDParse.unparse(rev._id.buffer);
Bogdan-Crn marked this conversation as resolved.
Show resolved Hide resolved
const uploadedAt = new Date(rev.timestamp).getTime();
Bogdan-Crn marked this conversation as resolved.
Show resolved Hide resolved
const filename = rev.rFile[0];

// check if the ref has the MD5 hash
const refEntry = await getRefEntry(teamspace, `${container}.history.ref`, filename);
Bogdan-Crn marked this conversation as resolved.
Show resolved Hide resolved

if (Object.keys(refEntry).includes('MD5Hash')) {
// if the ref has the hash create the object
returnValue = {
container,
code,
uploadedAt,
hash: refEntry.MD5Hash,
filename,
size: refEntry.size,
};
} else {
// if the ref does not have the hash get the file, create the hash, set the return object and update the ref with the hash
const file = await getFile(teamspace, `${container}.history`, filename);

const hash = CryptoJs.MD5(file).toString();

returnValue = {
container,
code,
uploadedAt,
hash,
filename,
size: refEntry.size,
};

await updateRef(teamspace, `${container}.history.ref`, { _id: filename }, { $set: { MD5Hash: hash } });
}

return returnValue;
};

module.exports = ModelList;
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

const { addModel, deleteModel, getModelList } = require('./commons/modelList');
const { addModel, deleteModel, getModelList, getModelMD5Hash } = require('./commons/modelList');
const { appendFavourites, deleteFavourites } = require('./commons/favourites');
const { getContainerById, getContainers, updateModelSettings } = require('../../../../models/modelSettings');
const { getLatestRevision, getRevisionByIdOrTag, getRevisionCount, getRevisionFormat, getRevisions, updateRevisionStatus } = require('../../../../models/revisions');
Expand Down Expand Up @@ -133,4 +133,9 @@ Containers.updateSettings = updateModelSettings;
Containers.getSettings = (teamspace, container) => getContainerById(teamspace,
container, { corID: 0, account: 0, permissions: 0 });

Containers.getRevisionMD5Hash = async (teamspace, container, revision, user) => {
Bogdan-Crn marked this conversation as resolved.
Show resolved Hide resolved
const response = await getModelMD5Hash(teamspace, container, revision, user);
return response;
};

module.exports = Containers;
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,14 @@

const { addModel, deleteModel, getModelList } = require('./commons/modelList');
const { appendFavourites, deleteFavourites } = require('./commons/favourites');
const { getFederationById, getFederations, updateModelSettings } = require('../../../../models/modelSettings');
const { getContainers, getFederationById, getFederations, updateModelSettings } = require('../../../../models/modelSettings');
const Comments = require('./commons/tickets.comments');
const Groups = require('./commons/groups');
const TicketGroups = require('./commons/tickets.groups');
const Tickets = require('./commons/tickets');
const Views = require('./commons/views');
const { getLatestRevision } = require('../../../../models/revisions');
const { getModelMD5Hash } = require('./commons/modelList');
const { getOpenTicketsCount } = require('./commons/tickets');
const { getProjectById } = require('../../../../models/projectSettings');
const { modelTypes } = require('../../../../models/modelSettings.constants');
Expand Down Expand Up @@ -75,7 +76,7 @@ const getLastUpdatesFromModels = async (teamspace, models) => {
}

return lastUpdates.length ? lastUpdates.sort((a, b) => b.timestamp
- a.timestamp)[0].timestamp : undefined;
- a.timestamp)[0].timestamp : undefined;
};

Federations.getFederationStats = async (teamspace, project, federation) => {
Expand Down Expand Up @@ -107,4 +108,23 @@ Federations.updateSettings = updateModelSettings;
Federations.getSettings = (teamspace, federation) => getFederationById(teamspace,
federation, { corID: 0, account: 0, permissions: 0, subModels: 0, federate: 0 });

Federations.getMD5Hash = async (teamspace, federation, user) => {
const { subModels: containers } = await getFederationById(teamspace, federation, { subModels: 1 });
const containerWithMetadata = await getContainers(
teamspace,
containers.map((container) => container._id),
{ _id: 1, name: 1, permissions: 1 });

const listOfPromises = containerWithMetadata.map(
Bogdan-Crn marked this conversation as resolved.
Show resolved Hide resolved
(container) => getModelMD5Hash(teamspace, container._id, null, user));

const promiseResponses = await Promise.allSettled(listOfPromises);
const responses = promiseResponses
Bogdan-Crn marked this conversation as resolved.
Show resolved Hide resolved
// make sure the promise is fulfilled and the value is not undefined or empty
.filter((response) => response.status === 'fulfilled' && response.value !== undefined)
.map((response) => response.value);

return responses;
};

module.exports = Federations;
Loading
Loading