Skip to content

Commit

Permalink
debug
Browse files Browse the repository at this point in the history
  • Loading branch information
@benzekrimaha
benzekrimaha committed Jan 22, 2025
1 parent c4fb3be commit 40c3696
Show file tree
Hide file tree
Showing 3 changed files with 53 additions and 11 deletions.
5 changes: 4 additions & 1 deletion .eslintrc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,10 @@
"no-lonely-if": 0,
"max-classes-per-file": 0,
"prefer-spread": 0,
"no-constructor-return": 0
"no-constructor-return": 0,
"no-console": 0,
"object-curly-spacing": 0,
"no-extend-native": 0
},
"globals": {
"BigInt": "readonly"
Expand Down
35 changes: 28 additions & 7 deletions tests/functional/utils/S3UtilsMongoClient.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,10 @@ const variations = [

const bucketMD = BucketInfo.fromObj(testBucketMD);
const BUCKET_NAME = bucketMD.getName();
console.log('BUCKET NAME ', BUCKET_NAME);
const USERSBUCKET = '__usersbucket';
const BUCKET_CREATE_DATE = new Date(testBucketCreationDate).getTime();


describe('S3UtilsMongoClient::getObjectMDStats', () => {
let client;
let repl;
Expand Down Expand Up @@ -297,15 +297,14 @@ describe('S3UtilsMongoClient::getObjectMDStats', () => {

afterEach(done => client.deleteBucket(BUCKET_NAME, logger, done));

it(`Should get correct results ${variation.it}`, done => {
it.only(`Should get correct results ${variation.it}`, done => {
const expected = {
versions: BigInt(1),
objects: BigInt(2),
dataManaged: {
locations: { 'us-east-1': { curr: BigInt(20), prev: BigInt(10) } },
total: { curr: BigInt(20), prev: BigInt(10) },
locations: { 'us-east-1': { curr: BigInt(20), prev: BigInt(10) } },
},
objects: BigInt(2),
stalled: 0,
versions: BigInt(1),
dataMetrics: {
account: {
[testAccountCanonicalId]: {
Expand Down Expand Up @@ -416,6 +415,7 @@ describe('S3UtilsMongoClient::getObjectMDStats', () => {
},
},
},
stalled: 0,
};
return async.series([
next => client.putObject(
Expand Down Expand Up @@ -443,15 +443,36 @@ describe('S3UtilsMongoClient::getObjectMDStats', () => {
next,
), // put object2
], () => client.getBucketAttributes(BUCKET_NAME, logger, (err, bucketInfo) => {
console.log('HERE THE LOG TO DEBUG 1 ', {bucketInfo, err});
assert.deepStrictEqual(err, null);
return client.getObjectMDStats(
BUCKET_NAME,
bucketInfo,
false,
logger,
(err, data) => {
console.log('HERE THE LOG TO DEBUG', {data, expected});
assert.deepStrictEqual(err, null);
assert.deepStrictEqual(data, expected);
// assert.deepStrictEqual(data, expected);\
// Compare each section separately to avoid circular reference issues
assert.strictEqual(data.versions.toString(), expected.versions.toString());
assert.strictEqual(data.objects.toString(), expected.objects.toString());
assert.strictEqual(data.stalled, expected.stalled);

// Compare dataManaged section
assert.deepStrictEqual(
JSON.parse(JSON.stringify(data.dataManaged)),
JSON.parse(JSON.stringify(expected.dataManaged)),
);

// Compare dataMetrics sections separately
const sections = ['account', 'bucket', 'location'];
sections.forEach(section => {
assert.deepStrictEqual(
JSON.parse(JSON.stringify(data.dataMetrics[section])),
JSON.parse(JSON.stringify(expected.dataMetrics[section])),
);
});
return done();
},
);
Expand Down
24 changes: 21 additions & 3 deletions utils/S3UtilsMongoClient.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ const __COUNT_ITEMS = 'countitems';

const bigIntMax = (...args) => args.reduce((m, e) => (e > m ? BigInt(e) : BigInt(m)));

BigInt.prototype.toJSON = () => this.toString();

const baseMetricsObject = {
masterCount: BigInt(0),
masterData: BigInt(0),
Expand Down Expand Up @@ -63,7 +65,10 @@ class S3UtilsMongoClient extends MongoClientInterface {
'value.creationDate': 1,
},
});
const TEST = await this.getCollection(USERSBUCKET).find({});
console.log('TEST', TEST);
const usersBucketCreationDatesArray = await cursorUsersBucketCreationDates.toArray();
console.log('USERS BUCKET CREATION DATES ARRAY', usersBucketCreationDatesArray);
return usersBucketCreationDatesArray
.reduce((map, obj) => ({ ...map, [obj._id]: obj.value.creationDate }), {});
} catch (err) {
Expand Down Expand Up @@ -164,8 +169,10 @@ class S3UtilsMongoClient extends MongoClientInterface {
async getObjectMDStats(bucketName, bucketInfo, isTransient, log, callback) {
let cursor;
let cursorMpuBucket;
console.log('WE ARE HEEERE', bucketName);
try {
const c = this.getCollection(bucketName);
console.log('HERE C ', c);
cursor = c.find({}, {
projection: {
'_id': 1,
Expand All @@ -182,6 +189,7 @@ class S3UtilsMongoClient extends MongoClientInterface {
'value.isPHD': 1,
},
});
console.log('CURSOR', cursor);
const collRes = {
bucket: {}, // bucket level metrics
location: {}, // location level metrics
Expand All @@ -197,20 +205,24 @@ class S3UtilsMongoClient extends MongoClientInterface {
const locationConfig = getLocationConfig(log);

const usersBucketCreationDatesMap = await this._getUsersBucketCreationDates(log);

console.log('USERS BUCKET CREATION DATES MAP', usersBucketCreationDatesMap);
const bucketStatus = bucketInfo.getVersioningConfiguration();
const isVer = (bucketStatus && (bucketStatus.Status === 'Enabled'
|| bucketStatus.Status === 'Suspended'));

if (!usersBucketCreationDatesMap) {
console.log('ERRORS INTERNAL', usersBucketCreationDatesMap);
return callback(errors.InternalError);
}

const bucketDate = usersBucketCreationDatesMap[`${bucketInfo.getOwner()}${constants.splitter}${bucketName}`];
console.log('BUCKET DATE', bucketDate);
if (bucketDate) {
bucketKey = `bucket_${bucketName}_${new Date(bucketDate).getTime()}`;
console.log('BUCKET KEY', bucketKey);
if (bucketKey) {
inflightsPreScan = await this.readStorageConsumptionInflights(bucketKey, log);
console.log('INGLIGHTS PRE SCAN', inflightsPreScan);
}
}

Expand Down Expand Up @@ -456,10 +468,13 @@ class S3UtilsMongoClient extends MongoClientInterface {
isRestoring: false,
isRestored: false,
}) {
console.log('HEEEEEERE 2');
if (!bucketName) {
return { error: new Error('no bucket name provided') };
}

console.log('bucketCreationDate', bucketCreationDate);

if (entry.value.isPHD) {
// PHD are created transiently in place of a master when it is deleted, until
// they get replaced with the "new" master. They may either hold no information
Expand All @@ -468,6 +483,8 @@ class S3UtilsMongoClient extends MongoClientInterface {
return {};
}

console.log('entry.value', entry.value);

const size = Number.parseInt(entry.value['content-length'], 10);
if (Number.isNaN(size)) {
return { error: new Error('invalid content length') };
Expand Down Expand Up @@ -530,7 +547,7 @@ class S3UtilsMongoClient extends MongoClientInterface {
delete results.location[location];
}
}

console.log('DATAAAAAA RESULTS', results);
return { data: results };
}

Expand Down Expand Up @@ -756,7 +773,7 @@ class S3UtilsMongoClient extends MongoClientInterface {
});

return {
versions: totalNonCurrentCount + totalNonCurrentColdCount + totalVersionRestoringCount + totalVerionsRestoredCount,
versions: bigIntMax(0, totalNonCurrentCount + totalNonCurrentColdCount + totalVersionRestoringCount + totalVerionsRestoredCount),
objects: totalCurrentCount + totalCurrentColdCount + totalRestoringCount + totalRestoredCount,
dataManaged: {
total: totalBytes,
Expand Down Expand Up @@ -805,6 +822,7 @@ class S3UtilsMongoClient extends MongoClientInterface {
}

static convertNumberToLong(obj) {
console.log('ON EST ICIIII SA RACE');
if (typeof obj !== 'object' || obj === null) {
return obj;
}
Expand Down

0 comments on commit 40c3696

Please sign in to comment.