Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[bug] fix s3 full match prefix #492

Open
wants to merge 2 commits into
base: master
Choose a base branch
from

Conversation

IMSupperkaka
Copy link

aws s3 and fs file systems behave inconsistently.

filesCount = yield* downloadFileFromStorage(ctx, cmd.getDocId() + cmd.getSaveKey(), changesDir);

yield* downloadFileFromStorage(ctx, cmd.getDocId() + cmd.getSaveKey(), tempDirs.source);

async function listObjects(storageCfg, strPath) {
let params = {
Bucket: storageCfg.bucketName,
Prefix: getFilePath(storageCfg, strPath)
};
let output = [];
await listObjectsExec(storageCfg, output, params);
return output;
}

e.g.

cmd.getDocId() + cmd.getSaveKey(); // RAPp9zLIAPNideSp + _860

// s3 storage objects
const objects = [
        'editfiles/data/RAPp9zLIAPNideSp_860',
        'editfiles/data/RAPp9zLIAPNideSp_860/changesHistory.json',
        'editfiles/data/RAPp9zLIAPNideSp_860/output.docx',
        'editfiles/data/RAPp9zLIAPNideSp_860/changes.zip',
        'editfiles/data/RAPp9zLIAPNideSp_8603',
        'editfiles/data/RAPp9zLIAPNideSp_8603/changesHistory.json',
        'editfiles/data/RAPp9zLIAPNideSp_8603/output.docx',
        'editfiles/data/RAPp9zLIAPNideSp_8603/changes.zip',
        'editfiles/data/RAPp9zLIAPNideSp'
    ];

Before:

const list = storage.listObjects(cfg, 'editfiles/data/RAPp9zLIAPNideSp_860');
// Print list: 
// [
//        'editfiles/data/RAPp9zLIAPNideSp_860',
//        'editfiles/data/RAPp9zLIAPNideSp_860/changesHistory.json',
//        'editfiles/data/RAPp9zLIAPNideSp_860/output.docx',
//        'editfiles/data/RAPp9zLIAPNideSp_860/changes.zip',
//        'editfiles/data/RAPp9zLIAPNideSp_8603',
//        'editfiles/data/RAPp9zLIAPNideSp_8603/changesHistory.json',
//        'editfiles/data/RAPp9zLIAPNideSp_8603/output.docx',
//        'editfiles/data/RAPp9zLIAPNideSp_8603/changes.zip',
// ]

After:

const list = storage.listObjects(cfg, 'editfiles/data/RAPp9zLIAPNideSp_860');
// Print list: 
// [
//        'editfiles/data/RAPp9zLIAPNideSp_860',
//        'editfiles/data/RAPp9zLIAPNideSp_860/changesHistory.json',
//        'editfiles/data/RAPp9zLIAPNideSp_860/output.docx',
//        'editfiles/data/RAPp9zLIAPNideSp_860/changes.zip',
// ]

Issues that the previous code may cause:

//make dirs
for (var i = 0; i < dirsToCreate.length; ++i) {
fs.mkdirSync(dirsToCreate[i]);
}

function receiveTask(data, ack) {
return co(function* () {
var res = null;
var task = null;
let outParams = {isAck: false};
let timeoutId = undefined;
let ctx = new operationContext.Context();
try {
task = new commonDefines.TaskQueueData(JSON.parse(data));
if (task) {
ctx.initFromTaskQueueData(task);
yield ctx.initTenantCache();
timeoutId = receiveTaskSetTimeout(ctx, task, ack, outParams);
res = yield* ExecuteTask(ctx, task);
}
} catch (err) {
ctx.logger.error('receiveTask %s', err.stack);
} finally {
clearTimeout(timeoutId);
if (!outParams.isAck) {
yield ackTask(ctx, res, task, ack);
}
}
});
}

receiveTask  {"errno":-17,"syscall":"mkdir","code":"EEXIST","path":"/tmp/ASC_CONVERT2025121-rr3tge.qow3/source"}

@CLAassistant
Copy link

CLAassistant commented Feb 24, 2025

CLA assistant check
All committers have signed the CLA.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
None yet
Projects
None yet
Development

Successfully merging this pull request may close these issues.

2 participants