diff options
author | Romulus Urakagi Tsai <urakagi@gmail.com> | 2020-05-05 14:08:36 +0800 |
---|---|---|
committer | Romulus Urakagi Tsai <urakagi@gmail.com> | 2020-05-05 14:08:36 +0800 |
commit | 0a1bfd37b3b1b2f4108a734c0449c1bd5a1b691f (patch) | |
tree | b1c21bbad460aca64808dd5ede107e4d32c5bb3f | |
parent | 5899b9366c23f31149e9de8ed006a7e30b4830d5 (diff) | |
download | wekan-0a1bfd37b3b1b2f4108a734c0449c1bd5a1b691f.tar.gz wekan-0a1bfd37b3b1b2f4108a734c0449c1bd5a1b691f.tar.bz2 wekan-0a1bfd37b3b1b2f4108a734c0449c1bd5a1b691f.zip |
Migrating attachments
-rw-r--r-- | server/migrations.js | 38 | ||||
-rw-r--r-- | server/old-attachments-migration.js | 212 |
2 files changed, 247 insertions, 3 deletions
diff --git a/server/migrations.js b/server/migrations.js index 3861f227..33f061f5 100644 --- a/server/migrations.js +++ b/server/migrations.js @@ -1039,9 +1039,41 @@ Migrations.add('fix-incorrect-dates', () => { console.log('cas', cas); }); +import { MongoInternals } from 'meteor/mongo'; + Migrations.add('change-attachment-library', () => { - console.log('migration called here'); - Migrations.rollback('change-attachment-library'); - console.log('migration rollbacked'); + const http = require('http'); + const fs = require('fs'); + CFSAttachments.find().forEach(file => { + const bucket = new MongoInternals.NpmModule.GridFSBucket(MongoInternals.defaultRemoteCollectionDriver().mongo.db, {bucketName: 'cfs_gridfs.attachments'}); + const gfsId = new MongoInternals.NpmModule.ObjectID(file.copies.attachments.key); + const reader = bucket.openDownloadStream(gfsId); + const path = `/var/attachments/${file.name()}`; + const fd = fs.createWriteStream(path); + reader.pipe(fd); + let opts = { + fileName: file.name(), + type: file.type(), + fileId: file._id, + meta: { + userId: file.userId, + boardId: file.boardId, + cardId: file.cardId + } + }; + if (file.listId) { + opts.meta.listId = file.listId; + } + if (file.swimlaneId) { + opts.meta.swimlaneId = file.swimlaneId; + } + Attachments.addFile(path, opts, (err, fileRef) => { + if (err) { + console.log('error when migrating ', fileName, err); + } else { + file.remove(); + } + }); + }); }); diff --git a/server/old-attachments-migration.js b/server/old-attachments-migration.js new file mode 100644 index 00000000..3a6aa85d --- /dev/null +++ b/server/old-attachments-migration.js @@ -0,0 +1,212 @@ +const localFSStore = process.env.ATTACHMENTS_STORE_PATH; +const storeName = 'attachments'; +const defaultStoreOptions = { + beforeWrite: fileObj => { + if (!fileObj.isImage()) { + return { + type: 'application/octet-stream', + }; + } + return {}; + }, +}; +let store; +if (localFSStore) { + // have to reinvent methods from FS.Store.GridFS and FS.Store.FileSystem + const fs = Npm.require('fs'); + const path = Npm.require('path'); + const mongodb = Npm.require('mongodb'); + const Grid = Npm.require('gridfs-stream'); + // calulate the absolute path here, because FS.Store.FileSystem didn't expose the aboslutepath or FS.Store didn't expose api calls :( + let pathname = localFSStore; + /*eslint camelcase: ["error", {allow: ["__meteor_bootstrap__"]}] */ + + if (!pathname && __meteor_bootstrap__ && __meteor_bootstrap__.serverDir) { + pathname = path.join( + __meteor_bootstrap__.serverDir, + `../../../cfs/files/${storeName}`, + ); + } + + if (!pathname) + throw new Error('FS.Store.FileSystem unable to determine path'); + + // Check if we have '~/foo/bar' + if (pathname.split(path.sep)[0] === '~') { + const homepath = + process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE; + if (homepath) { + pathname = pathname.replace('~', homepath); + } else { + throw new Error('FS.Store.FileSystem unable to resolve "~" in path'); + } + } + + // Set absolute path + const absolutePath = path.resolve(pathname); + + const _FStore = new FS.Store.FileSystem(storeName, { + path: localFSStore, + ...defaultStoreOptions, + }); + const GStore = { + fileKey(fileObj) { + const key = { + _id: null, + filename: null, + }; + + // If we're passed a fileObj, we retrieve the _id and filename from it. + if (fileObj) { + const info = fileObj._getInfo(storeName, { + updateFileRecordFirst: false, + }); + key._id = info.key || null; + key.filename = + info.name || + fileObj.name({ updateFileRecordFirst: false }) || + `${fileObj.collectionName}-${fileObj._id}`; + } + + // If key._id is null at this point, createWriteStream will let GridFS generate a new ID + return key; + }, + db: undefined, + mongoOptions: { useNewUrlParser: true }, + mongoUrl: process.env.MONGO_URL, + init() { + this._init(err => { + this.inited = !err; + }); + }, + _init(callback) { + const self = this; + mongodb.MongoClient.connect(self.mongoUrl, self.mongoOptions, function( + err, + db, + ) { + if (err) { + return callback(err); + } + self.db = db; + return callback(null); + }); + return; + }, + createReadStream(fileKey, options) { + const self = this; + if (!self.inited) { + self.init(); + return undefined; + } + options = options || {}; + + // Init GridFS + const gfs = new Grid(self.db, mongodb); + + // Set the default streamning settings + const settings = { + _id: new mongodb.ObjectID(fileKey._id), + root: `cfs_gridfs.${storeName}`, + }; + + // Check if this should be a partial read + if ( + typeof options.start !== 'undefined' && + typeof options.end !== 'undefined' + ) { + // Add partial info + settings.range = { + startPos: options.start, + endPos: options.end, + }; + } + return gfs.createReadStream(settings); + }, + }; + GStore.init(); + const CRS = 'createReadStream'; + const _CRS = `_${CRS}`; + const FStore = _FStore._transform; + FStore[_CRS] = FStore[CRS].bind(FStore); + FStore[CRS] = function(fileObj, options) { + let stream; + try { + const localFile = path.join( + absolutePath, + FStore.storage.fileKey(fileObj), + ); + const state = fs.statSync(localFile); + if (state) { + stream = FStore[_CRS](fileObj, options); + } + } catch (e) { + // file is not there, try GridFS ? + stream = undefined; + } + if (stream) return stream; + else { + try { + const stream = GStore[CRS](GStore.fileKey(fileObj), options); + return stream; + } catch (e) { + return undefined; + } + } + }.bind(FStore); + store = _FStore; +} else { + store = new FS.Store.GridFS(localFSStore ? `G${storeName}` : storeName, { + // XXX Add a new store for cover thumbnails so we don't load big images in + // the general board view + // If the uploaded document is not an image we need to enforce browser + // download instead of execution. This is particularly important for HTML + // files that the browser will just execute if we don't serve them with the + // appropriate `application/octet-stream` MIME header which can lead to user + // data leaks. I imagine other formats (like PDF) can also be attack vectors. + // See https://github.com/wekan/wekan/issues/99 + // XXX Should we use `beforeWrite` option of CollectionFS instead of + // collection-hooks? + // We should use `beforeWrite`. + ...defaultStoreOptions, + }); +} +CFSAttachments = new FS.Collection('attachments', { + stores: [store], +}); + +if (Meteor.isServer) { + Meteor.startup(() => { + CFSAttachments.files._ensureIndex({ cardId: 1 }); + }); + + CFSAttachments.allow({ + insert(userId, doc) { + return allowIsBoardMember(userId, Boards.findOne(doc.boardId)); + }, + update(userId, doc) { + return allowIsBoardMember(userId, Boards.findOne(doc.boardId)); + }, + remove(userId, doc) { + return allowIsBoardMember(userId, Boards.findOne(doc.boardId)); + }, + // We authorize the attachment download either: + // - if the board is public, everyone (even unconnected) can download it + // - if the board is private, only board members can download it + download(userId, doc) { + if (Meteor.isServer) { + return true; + } + const board = Boards.findOne(doc.boardId); + if (board.isPublic()) { + return true; + } else { + return board.hasMember(userId); + } + }, + + fetch: ['boardId'], + }); +} + +export default CFSAttachments; |