diff options
-rw-r--r-- | .meteor/packages | 1 | ||||
-rw-r--r-- | .meteor/versions | 1 | ||||
-rw-r--r-- | client/components/activities/activities.js | 12 | ||||
-rw-r--r-- | client/components/cards/attachments.jade | 13 | ||||
-rw-r--r-- | client/components/cards/attachments.js | 103 | ||||
-rw-r--r-- | client/components/cards/attachments.styl | 11 | ||||
-rw-r--r-- | client/components/cards/minicard.jade | 2 | ||||
-rw-r--r-- | client/components/cards/minicard.js | 3 | ||||
-rwxr-xr-x | client/components/main/editor.js | 50 | ||||
-rw-r--r-- | client/lib/utils.js | 44 | ||||
-rw-r--r-- | models/activities.js | 2 | ||||
-rw-r--r-- | models/attachments.js | 342 | ||||
-rw-r--r-- | models/cards.js | 21 | ||||
-rw-r--r-- | models/export.js | 240 | ||||
-rw-r--r-- | models/trelloCreator.js | 1 | ||||
-rw-r--r-- | models/wekanCreator.js | 2 | ||||
-rw-r--r-- | sandstorm-pkgdef.capnp | 7 | ||||
-rw-r--r-- | server/migrations.js | 45 | ||||
-rw-r--r-- | server/old-attachments-migration.js | 212 | ||||
-rw-r--r-- | server/publications/boards.js | 2 | ||||
-rwxr-xr-x | snap-src/bin/config | 4 | ||||
-rwxr-xr-x | snap-src/bin/mongodb-control | 8 |
22 files changed, 377 insertions, 749 deletions
diff --git a/.meteor/packages b/.meteor/packages index b0df2be8..ba278f34 100644 --- a/.meteor/packages +++ b/.meteor/packages @@ -98,4 +98,3 @@ percolate:synced-cron easylogic:summernote cfs:filesystem ostrio:cookies -ostrio:files diff --git a/.meteor/versions b/.meteor/versions index 6b6842ad..722547de 100644 --- a/.meteor/versions +++ b/.meteor/versions @@ -134,7 +134,6 @@ observe-sequence@1.0.16 ongoworks:speakingurl@1.1.0 ordered-dict@1.1.0 ostrio:cookies@2.6.0 -ostrio:files@1.14.2 peerlibrary:assert@0.3.0 peerlibrary:base-component@0.16.0 peerlibrary:blaze-components@0.15.1 diff --git a/client/components/activities/activities.js b/client/components/activities/activities.js index 4f15d421..5d356f6e 100644 --- a/client/components/activities/activities.js +++ b/client/components/activities/activities.js @@ -151,23 +151,21 @@ BlazeComponent.extendComponent({ }, attachmentLink() { - const activity = this.currentData().activity; - const attachment = activity.attachment(); - const link = attachment ? attachment.link('original', '/') : null; + const attachment = this.currentData().activity.attachment(); // trying to display url before file is stored generates js errors return ( (attachment && - link && + attachment.url({ download: true }) && Blaze.toHTML( HTML.A( { - href: link, + href: attachment.url({ download: true }), target: '_blank', }, - attachment.name, + attachment.name(), ), )) || - activity.attachmentName + this.currentData().activity.attachmentName ); }, diff --git a/client/components/cards/attachments.jade b/client/components/cards/attachments.jade index eda6d118..61454fa7 100644 --- a/client/components/cards/attachments.jade +++ b/client/components/cards/attachments.jade @@ -18,19 +18,12 @@ template(name="attachmentDeletePopup") p {{_ "attachment-delete-pop"}} button.js-confirm.negate.full(type="submit") {{_ 'delete'}} -template(name="uploadingPopup") - .uploading-info - span.upload-percentage {{progress}}% - .upload-progress-frame - .upload-progress-bar(style="width: {{progress}}%;") - span.upload-size {{fileSize}} - template(name="attachmentsGalery") .attachments-galery each attachments .attachment-item - a.attachment-thumbnail.swipebox(href="{{url}}" download="{{name}}" title="{{name}}") - if isUploaded + a.attachment-thumbnail.swipebox(href="{{url}}" title="{{name}}") + if isUploaded if isImage img.attachment-thumbnail-img(src="{{url}}") else @@ -40,7 +33,7 @@ template(name="attachmentsGalery") p.attachment-details = name span.attachment-details-actions - a.js-download(href="{{url download=true}}" download="{{name}}") + a.js-download(href="{{url download=true}}") i.fa.fa-download | {{_ 'download'}} if currentUser.isBoardMember diff --git a/client/components/cards/attachments.js b/client/components/cards/attachments.js index 81f6c6e1..e4439155 100644 --- a/client/components/cards/attachments.js +++ b/client/components/cards/attachments.js @@ -13,10 +13,10 @@ Template.attachmentsGalery.events({ event.stopPropagation(); }, 'click .js-add-cover'() { - Cards.findOne(this.meta.cardId).setCover(this._id); + Cards.findOne(this.cardId).setCover(this._id); }, 'click .js-remove-cover'() { - Cards.findOne(this.meta.cardId).unsetCover(); + Cards.findOne(this.cardId).unsetCover(); }, 'click .js-preview-image'(event) { Popup.open('previewAttachedImage').call(this, event); @@ -45,63 +45,22 @@ Template.attachmentsGalery.events({ }, }); -Template.attachmentsGalery.helpers({ - url() { - return Attachments.link(this, 'original', '/'); - }, - isUploaded() { - return !this.meta.uploading; - }, - isImage() { - return !!this.isImage; - }, -}); - Template.previewAttachedImagePopup.events({ 'click .js-large-image-clicked'() { Popup.close(); }, }); -Template.previewAttachedImagePopup.helpers({ - url() { - return Attachments.link(this, 'original', '/'); - } -}); - -// For uploading popup - -let uploadFileSize = new ReactiveVar(''); -let uploadProgress = new ReactiveVar(0); - Template.cardAttachmentsPopup.events({ - 'change .js-attach-file'(event, instance) { + 'change .js-attach-file'(event) { const card = this; - const callbacks = { - onBeforeUpload: (err, fileData) => { - Popup.open('uploading')(this.clickEvent); - uploadFileSize.set('...'); - uploadProgress.set(0); - return true; - }, - onUploaded: (err, attachment) => { - if (attachment && attachment._id && attachment.isImage) { - card.setCover(attachment._id); - } - Popup.close(); - }, - onStart: (error, fileData) => { - uploadFileSize.set(formatBytes(fileData.size)); - }, - onError: (err, fileObj) => { - console.log('Error!', err); - }, - onProgress: (progress, fileData) => { - uploadProgress.set(progress); - } - }; const processFile = f => { - Utils.processUploadedAttachment(card, f, callbacks); + Utils.processUploadedAttachment(card, f, attachment => { + if (attachment && attachment._id && attachment.isImage()) { + card.setCover(attachment._id); + } + Popup.close(); + }); }; FS.Utility.eachFile(event, f => { @@ -141,22 +100,12 @@ Template.cardAttachmentsPopup.events({ }); }, 'click .js-computer-upload'(event, templateInstance) { - this.clickEvent = event; templateInstance.find('.js-attach-file').click(); event.preventDefault(); }, 'click .js-upload-clipboard-image': Popup.open('previewClipboardImage'), }); -Template.uploadingPopup.helpers({ - fileSize: () => { - return uploadFileSize.get(); - }, - progress: () => { - return uploadProgress.get(); - } -}); - const MAX_IMAGE_PIXEL = Utils.MAX_IMAGE_PIXEL; const COMPRESS_RATIO = Utils.IMAGE_COMPRESS_RATIO; let pastedResults = null; @@ -200,26 +149,20 @@ Template.previewClipboardImagePopup.events({ if (results && results.file) { window.oPasted = pastedResults; const card = this; - const settings = { - file: results.file, - streams: 'dynamic', - chunkSize: 'dynamic', - }; + const file = new FS.File(results.file); if (!results.name) { // if no filename, it's from clipboard. then we give it a name, with ext name from MIME type if (typeof results.file.type === 'string') { - settings.fileName = - new Date().getTime() + results.file.type.replace('.+/', ''); + file.name(results.file.type.replace('image/', 'clipboard.')); } } - settings.meta = {}; - settings.meta.updatedAt = new Date().getTime(); - settings.meta.boardId = card.boardId; - settings.meta.cardId = card._id; - settings.meta.userId = Meteor.userId(); - const attachment = Attachments.insert(settings); + file.updatedAt(new Date()); + file.boardId = card.boardId; + file.cardId = card._id; + file.userId = Meteor.userId(); + const attachment = Attachments.insert(file); - if (attachment && attachment._id && attachment.isImage) { + if (attachment && attachment._id && attachment.isImage()) { card.setCover(attachment._id); } @@ -229,15 +172,3 @@ Template.previewClipboardImagePopup.events({ } }, }); - -function formatBytes(bytes, decimals = 2) { - if (bytes === 0) return '0 Bytes'; - - const k = 1024; - const dm = decimals < 0 ? 0 : decimals; - const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']; - - const i = Math.floor(Math.log(bytes) / Math.log(k)); - - return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i]; -} diff --git a/client/components/cards/attachments.styl b/client/components/cards/attachments.styl index 61ea8232..4a22fd8a 100644 --- a/client/components/cards/attachments.styl +++ b/client/components/cards/attachments.styl @@ -64,17 +64,6 @@ border: 1px solid black box-shadow: 0 1px 2px rgba(0,0,0,.2) -.uploading-info - .upload-progress-frame - background-color: grey; - border: 1px solid; - height: 22px; - - .upload-progress-bar - background-color: blue; - height: 20px; - padding: 1px; - @media screen and (max-width: 800px) .attachments-galery flex-direction diff --git a/client/components/cards/minicard.jade b/client/components/cards/minicard.jade index e9de2fea..79dd9127 100644 --- a/client/components/cards/minicard.jade +++ b/client/components/cards/minicard.jade @@ -11,7 +11,7 @@ template(name="minicard") .handle .fa.fa-arrows if cover - .minicard-cover(style="background-image: url('{{coverUrl}}');") + .minicard-cover(style="background-image: url('{{cover.url}}');") if labels .minicard-labels each labels diff --git a/client/components/cards/minicard.js b/client/components/cards/minicard.js index 200e019d..da36b87f 100644 --- a/client/components/cards/minicard.js +++ b/client/components/cards/minicard.js @@ -52,7 +52,4 @@ Template.minicard.helpers({ return false; } }, - coverUrl() { - return Attachments.findOne(this.coverId).link('original', '/'); - }, }); diff --git a/client/components/main/editor.js b/client/components/main/editor.js index abe4160f..0c2e3186 100755 --- a/client/components/main/editor.js +++ b/client/components/main/editor.js @@ -152,31 +152,33 @@ Template.editor.onRendered(() => { const processData = function(fileObj) { Utils.processUploadedAttachment( currentCard, - fileObj, - { onUploaded: - attachment => { - if (attachment && attachment._id && attachment.isImage) { - attachment.one('uploaded', function() { - const maxTry = 3; - const checkItvl = 500; - let retry = 0; - const checkUrl = function() { - // even though uploaded event fired, attachment.url() is still null somehow //TODO - const url = Attachments.link(attachment, 'original', '/'); - if (url) { - insertImage( - `${location.protocol}//${location.host}${url}`, - ); - } else { - retry++; - if (retry < maxTry) { - setTimeout(checkUrl, checkItvl); - } + fileObj, + attachment => { + if ( + attachment && + attachment._id && + attachment.isImage() + ) { + attachment.one('uploaded', function() { + const maxTry = 3; + const checkItvl = 500; + let retry = 0; + const checkUrl = function() { + // even though uploaded event fired, attachment.url() is still null somehow //TODO + const url = attachment.url(); + if (url) { + insertImage( + `${location.protocol}//${location.host}${url}`, + ); + } else { + retry++; + if (retry < maxTry) { + setTimeout(checkUrl, checkItvl); } - }; - checkUrl(); - }); - } + } + }; + checkUrl(); + }); } }, ); diff --git a/client/lib/utils.js b/client/lib/utils.js index e72f177e..c921fddc 100644 --- a/client/lib/utils.js +++ b/client/lib/utils.js @@ -61,38 +61,30 @@ Utils = { }, MAX_IMAGE_PIXEL: Meteor.settings.public.MAX_IMAGE_PIXEL, COMPRESS_RATIO: Meteor.settings.public.IMAGE_COMPRESS_RATIO, - processUploadedAttachment(card, fileObj, callbacks) { + processUploadedAttachment(card, fileObj, callback) { + const next = attachment => { + if (typeof callback === 'function') { + callback(attachment); + } + }; if (!card) { - return onUploaded(); + return next(); } - let settings = { - file: fileObj, - streams: 'dynamic', - chunkSize: 'dynamic', - }; - settings.meta = { - uploading: true - }; + const file = new FS.File(fileObj); if (card.isLinkedCard()) { - settings.meta.boardId = Cards.findOne(card.linkedId).boardId; - settings.meta.cardId = card.linkedId; + file.boardId = Cards.findOne(card.linkedId).boardId; + file.cardId = card.linkedId; } else { - settings.meta.boardId = card.boardId; - settings.meta.swimlaneId = card.swimlaneId; - settings.meta.listId = card.listId; - settings.meta.cardId = card._id; + file.boardId = card.boardId; + file.swimlaneId = card.swimlaneId; + file.listId = card.listId; + file.cardId = card._id; } - settings.meta.userId = Meteor.userId(); - if (typeof callbacks === 'function') { - settings.onEnd = callbacks; - } else { - for (const key in callbacks) { - if (key.substring(0, 2) === 'on') { - settings[key] = callbacks[key]; - } - } + file.userId = Meteor.userId(); + if (file.original) { + file.original.name = fileObj.name; } - Attachments.insert(settings); + return next(Attachments.insert(file)); }, shrinkImage(options) { // shrink image to certain size diff --git a/models/activities.js b/models/activities.js index 2663dd29..b5fcb7d8 100644 --- a/models/activities.js +++ b/models/activities.js @@ -217,7 +217,7 @@ if (Meteor.isServer) { } if (activity.attachmentId) { const attachment = activity.attachment(); - params.attachment = attachment.name; + params.attachment = attachment.original.name; params.attachmentId = attachment._id; } if (activity.checklistId) { diff --git a/models/attachments.js b/models/attachments.js index 03999f55..3fe1d745 100644 --- a/models/attachments.js +++ b/models/attachments.js @@ -1,127 +1,263 @@ -import { FilesCollection } from 'meteor/ostrio:files'; -const fs = require('fs'); - -const collectionName = 'attachments2'; - -Attachments = new FilesCollection({ - storagePath: storagePath(), - debug: false, -// allowClientCode: true, - collectionName: 'attachments2', - onAfterUpload: onAttachmentUploaded, - onBeforeRemove: onAttachmentRemoving -}); +const localFSStore = process.env.ATTACHMENTS_STORE_PATH; +const storeName = 'attachments'; +const defaultStoreOptions = { + beforeWrite: fileObj => { + if (!fileObj.isImage()) { + return { + type: 'application/octet-stream', + }; + } + return {}; + }, +}; +let store; +if (localFSStore) { + // have to reinvent methods from FS.Store.GridFS and FS.Store.FileSystem + const fs = Npm.require('fs'); + const path = Npm.require('path'); + const mongodb = Npm.require('mongodb'); + const Grid = Npm.require('gridfs-stream'); + // calulate the absolute path here, because FS.Store.FileSystem didn't expose the aboslutepath or FS.Store didn't expose api calls :( + let pathname = localFSStore; + /*eslint camelcase: ["error", {allow: ["__meteor_bootstrap__"]}] */ -if (Meteor.isServer) { - Meteor.startup(() => { - Attachments.collection._ensureIndex({ cardId: 1 }); - }); + if (!pathname && __meteor_bootstrap__ && __meteor_bootstrap__.serverDir) { + pathname = path.join( + __meteor_bootstrap__.serverDir, + `../../../cfs/files/${storeName}`, + ); + } - // TODO: Permission related - Attachments.allow({ - insert() { - return false; - }, - update() { - return true; - }, - remove() { - return true; + if (!pathname) + throw new Error('FS.Store.FileSystem unable to determine path'); + + // Check if we have '~/foo/bar' + if (pathname.split(path.sep)[0] === '~') { + const homepath = + process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE; + if (homepath) { + pathname = pathname.replace('~', homepath); + } else { + throw new Error('FS.Store.FileSystem unable to resolve "~" in path'); } - }); + } - Meteor.methods({ - cloneAttachment(file, overrides) { - check(file, Object); - check(overrides, Match.Maybe(Object)); - const path = file.path; - const opts = { - fileName: file.name, - type: file.type, - meta: file.meta, - userId: file.userId + // Set absolute path + const absolutePath = path.resolve(pathname); + + const _FStore = new FS.Store.FileSystem(storeName, { + path: localFSStore, + ...defaultStoreOptions, + }); + const GStore = { + fileKey(fileObj) { + const key = { + _id: null, + filename: null, }; - for (let key in overrides) { - if (key === 'meta') { - for (let metaKey in overrides.meta) { - opts.meta[metaKey] = overrides.meta[metaKey]; - } - } else { - opts[key] = overrides[key]; - } + + // If we're passed a fileObj, we retrieve the _id and filename from it. + if (fileObj) { + const info = fileObj._getInfo(storeName, { + updateFileRecordFirst: false, + }); + key._id = info.key || null; + key.filename = + info.name || + fileObj.name({ updateFileRecordFirst: false }) || + `${fileObj.collectionName}-${fileObj._id}`; } - const buffer = fs.readFileSync(path); - Attachments.write(buffer, opts, (err, fileRef) => { + + // If key._id is null at this point, createWriteStream will let GridFS generate a new ID + return key; + }, + db: undefined, + mongoOptions: { useNewUrlParser: true }, + mongoUrl: process.env.MONGO_URL, + init() { + this._init(err => { + this.inited = !err; + }); + }, + _init(callback) { + const self = this; + mongodb.MongoClient.connect(self.mongoUrl, self.mongoOptions, function( + err, + db, + ) { if (err) { - console.log('Error when cloning record', err); + return callback(err); } + self.db = db; + return callback(null); }); - return true; + return; + }, + createReadStream(fileKey, options) { + const self = this; + if (!self.inited) { + self.init(); + return undefined; + } + options = options || {}; + + // Init GridFS + const gfs = new Grid(self.db, mongodb); + + // Set the default streamning settings + const settings = { + _id: new mongodb.ObjectID(fileKey._id), + root: `cfs_gridfs.${storeName}`, + }; + + // Check if this should be a partial read + if ( + typeof options.start !== 'undefined' && + typeof options.end !== 'undefined' + ) { + // Add partial info + settings.range = { + startPos: options.start, + endPos: options.end, + }; + } + return gfs.createReadStream(settings); + }, + }; + GStore.init(); + const CRS = 'createReadStream'; + const _CRS = `_${CRS}`; + const FStore = _FStore._transform; + FStore[_CRS] = FStore[CRS].bind(FStore); + FStore[CRS] = function(fileObj, options) { + let stream; + try { + const localFile = path.join( + absolutePath, + FStore.storage.fileKey(fileObj), + ); + const state = fs.statSync(localFile); + if (state) { + stream = FStore[_CRS](fileObj, options); + } + } catch (e) { + // file is not there, try GridFS ? + stream = undefined; } + if (stream) return stream; + else { + try { + const stream = GStore[CRS](GStore.fileKey(fileObj), options); + return stream; + } catch (e) { + return undefined; + } + } + }.bind(FStore); + store = _FStore; +} else { + store = new FS.Store.GridFS(localFSStore ? `G${storeName}` : storeName, { + // XXX Add a new store for cover thumbnails so we don't load big images in + // the general board view + // If the uploaded document is not an image we need to enforce browser + // download instead of execution. This is particularly important for HTML + // files that the browser will just execute if we don't serve them with the + // appropriate `application/octet-stream` MIME header which can lead to user + // data leaks. I imagine other formats (like PDF) can also be attack vectors. + // See https://github.com/wekan/wekan/issues/99 + // XXX Should we use `beforeWrite` option of CollectionFS instead of + // collection-hooks? + // We should use `beforeWrite`. + ...defaultStoreOptions, }); +} +Attachments = new FS.Collection('attachments', { + stores: [store], +}); - Meteor.publish(collectionName, function() { - return Attachments.find().cursor; +if (Meteor.isServer) { + Meteor.startup(() => { + Attachments.files._ensureIndex({ cardId: 1 }); }); -} else { - Meteor.subscribe(collectionName); -} -function storagePath(defaultPath) { - const storePath = process.env.ATTACHMENTS_STORE_PATH; - return storePath ? storePath : defaultPath; + Attachments.allow({ + insert(userId, doc) { + return allowIsBoardMember(userId, Boards.findOne(doc.boardId)); + }, + update(userId, doc) { + return allowIsBoardMember(userId, Boards.findOne(doc.boardId)); + }, + remove(userId, doc) { + return allowIsBoardMember(userId, Boards.findOne(doc.boardId)); + }, + // We authorize the attachment download either: + // - if the board is public, everyone (even unconnected) can download it + // - if the board is private, only board members can download it + download(userId, doc) { + const board = Boards.findOne(doc.boardId); + if (board.isPublic()) { + return true; + } else { + return board.hasMember(userId); + } + }, + + fetch: ['boardId'], + }); } -function onAttachmentUploaded(fileRef) { - Attachments.update({_id:fileRef._id}, {$set: {"meta.uploading": false}}); - if (!fileRef.meta.source || fileRef.meta.source !== 'import') { - // Add activity about adding the attachment +// XXX Enforce a schema for the Attachments CollectionFS + +if (Meteor.isServer) { + Attachments.files.after.insert((userId, doc) => { + // If the attachment doesn't have a source field + // or its source is different than import + if (!doc.source || doc.source !== 'import') { + // Add activity about adding the attachment + Activities.insert({ + userId, + type: 'card', + activityType: 'addAttachment', + attachmentId: doc._id, + // this preserves the name so that notifications can be meaningful after + // this file is removed + attachmentName: doc.original.name, + boardId: doc.boardId, + cardId: doc.cardId, + listId: doc.listId, + swimlaneId: doc.swimlaneId, + }); + } else { + // Don't add activity about adding the attachment as the activity + // be imported and delete source field + Attachments.update( + { + _id: doc._id, + }, + { + $unset: { + source: '', + }, + }, + ); + } + }); + + Attachments.files.before.remove((userId, doc) => { Activities.insert({ - userId: fileRef.userId, + userId, type: 'card', - activityType: 'addAttachment', - attachmentId: fileRef._id, + activityType: 'deleteAttachment', + attachmentId: doc._id, // this preserves the name so that notifications can be meaningful after - // this file is removed - attachmentName: fileRef.name, - boardId: fileRef.meta.boardId, - cardId: fileRef.meta.cardId, - listId: fileRef.meta.listId, - swimlaneId: fileRef.meta.swimlaneId, + // this file is removed + attachmentName: doc.original.name, + boardId: doc.boardId, + cardId: doc.cardId, + listId: doc.listId, + swimlaneId: doc.swimlaneId, }); - } else { - // Don't add activity about adding the attachment as the activity - // be imported and delete source field - Attachments.collection.update( - { - _id: fileRef._id, - }, - { - $unset: { - 'meta.source': '', - }, - }, - ); - } -} - -function onAttachmentRemoving(cursor) { - const file = cursor.get()[0]; - const meta = file.meta; - Activities.insert({ - userId: this.userId, - type: 'card', - activityType: 'deleteAttachment', - attachmentId: file._id, - // this preserves the name so that notifications can be meaningful after - // this file is removed - attachmentName: file.name, - boardId: meta.boardId, - cardId: meta.cardId, - listId: meta.listId, - swimlaneId: meta.swimlaneId, }); - return true; } export default Attachments; diff --git a/models/cards.js b/models/cards.js index 757772f3..6d5e23cc 100644 --- a/models/cards.js +++ b/models/cards.js @@ -412,14 +412,10 @@ Cards.helpers({ const _id = Cards.insert(this); // Copy attachments - oldCard.attachments().forEach((file) => { - Meteor.call('cloneAttachment', file, - { - meta: { - cardId: _id - } - } - ); + oldCard.attachments().forEach(att => { + att.cardId = _id; + delete att._id; + return Attachments.insert(att); }); // copy checklists @@ -522,15 +518,14 @@ Cards.helpers({ attachments() { if (this.isLinkedCard()) { return Attachments.find( - { 'meta.cardId': this.linkedId }, + { cardId: this.linkedId }, { sort: { uploadedAt: -1 } }, ); } else { - let ret = Attachments.find( - { 'meta.cardId': this._id }, + return Attachments.find( + { cardId: this._id }, { sort: { uploadedAt: -1 } }, ); - return ret; } }, @@ -539,7 +534,7 @@ Cards.helpers({ const cover = Attachments.findOne(this.coverId); // if we return a cover before it is fully stored, we will get errors when we try to display it // todo XXX we could return a default "upload pending" image in the meantime? - return cover && cover.link(); + return cover && cover.url() && cover; }, checklists() { diff --git a/models/export.js b/models/export.js index 1eb47e54..c3783679 100644 --- a/models/export.js +++ b/models/export.js @@ -1,3 +1,4 @@ +import { Exporter } from './exporter'; /* global JsonRoutes */ if (Meteor.isServer) { // todo XXX once we have a real API in place, move that route there @@ -7,10 +8,10 @@ if (Meteor.isServer) { // on the client instead of copy/pasting the route path manually between the // client and the server. /** - * @operation export + * @operation exportJson * @tag Boards * - * @summary This route is used to export the board. + * @summary This route is used to export the board to a json file format. * * @description If user is already logged-in, pass loginToken as param * "authToken": '/api/boards/:boardId/export?authToken=:token' @@ -46,199 +47,52 @@ if (Meteor.isServer) { JsonRoutes.sendResult(res, 403); } }); -} - -// exporter maybe is broken since Gridfs introduced, add fs and path - -export class Exporter { - constructor(boardId) { - this._boardId = boardId; - } - - build() { - const fs = Npm.require('fs'); - const os = Npm.require('os'); - const path = Npm.require('path'); - - const byBoard = { boardId: this._boardId }; - const byBoardNoLinked = { - boardId: this._boardId, - linkedId: { $in: ['', null] }, - }; - // we do not want to retrieve boardId in related elements - const noBoardId = { - fields: { - boardId: 0, - }, - }; - const result = { - _format: 'wekan-board-1.0.0', - }; - _.extend( - result, - Boards.findOne(this._boardId, { - fields: { - stars: 0, - }, - }), - ); - result.lists = Lists.find(byBoard, noBoardId).fetch(); - result.cards = Cards.find(byBoardNoLinked, noBoardId).fetch(); - result.swimlanes = Swimlanes.find(byBoard, noBoardId).fetch(); - result.customFields = CustomFields.find( - { boardIds: { $in: [this.boardId] } }, - { fields: { boardId: 0 } }, - ).fetch(); - result.comments = CardComments.find(byBoard, noBoardId).fetch(); - result.activities = Activities.find(byBoard, noBoardId).fetch(); - result.rules = Rules.find(byBoard, noBoardId).fetch(); - result.checklists = []; - result.checklistItems = []; - result.subtaskItems = []; - result.triggers = []; - result.actions = []; - result.cards.forEach(card => { - result.checklists.push( - ...Checklists.find({ - cardId: card._id, - }).fetch(), - ); - result.checklistItems.push( - ...ChecklistItems.find({ - cardId: card._id, - }).fetch(), - ); - result.subtaskItems.push( - ...Cards.find({ - parentId: card._id, - }).fetch(), - ); - }); - result.rules.forEach(rule => { - result.triggers.push( - ...Triggers.find( - { - _id: rule.triggerId, - }, - noBoardId, - ).fetch(), - ); - result.actions.push( - ...Actions.find( - { - _id: rule.actionId, - }, - noBoardId, - ).fetch(), - ); - }); - - // [Old] for attachments we only export IDs and absolute url to original doc - // [New] Encode attachment to base64 - - const getBase64Data = function(doc, callback) { - let buffer = Buffer.allocUnsafe(0); - buffer.fill(0); - - // callback has the form function (err, res) {} - const tmpFile = path.join( - os.tmpdir(), - `tmpexport${process.pid}${Math.random()}`, - ); - const tmpWriteable = fs.createWriteStream(tmpFile); - const readStream = fs.createReadStream(doc.path); - readStream.on('data', function(chunk) { - buffer = Buffer.concat([buffer, chunk]); - }); - - readStream.on('error', function(err) { - callback(null, null); - }); - readStream.on('end', function() { - // done - fs.unlink(tmpFile, () => { - //ignored - }); - callback(null, buffer.toString('base64')); + /** + * @operation exportCSV/TSV + * @tag Boards + * + * @summary This route is used to export the board to a CSV or TSV file format. + * + * @description If user is already logged-in, pass loginToken as param + * + * See https://blog.kayla.com.au/server-side-route-authentication-in-meteor/ + * for detailed explanations + * + * @param {string} boardId the ID of the board we are exporting + * @param {string} authToken the loginToken + * @param {string} delimiter delimiter to use while building export. Default is comma ',' + */ + Picker.route('/api/boards/:boardId/export/csv', function(params, req, res) { + const boardId = params.boardId; + let user = null; + const loginToken = params.query.authToken; + if (loginToken) { + const hashToken = Accounts._hashLoginToken(loginToken); + user = Meteor.users.findOne({ + 'services.resume.loginTokens.hashedToken': hashToken, }); - readStream.pipe(tmpWriteable); - }; - const getBase64DataSync = Meteor.wrapAsync(getBase64Data); - result.attachments = Attachments.find({ 'meta.boardId': byBoard.boardId }) - .fetch() - .map(attachment => { - let filebase64 = null; - filebase64 = getBase64DataSync(attachment); - - return { - _id: attachment._id, - cardId: attachment.meta.cardId, - //url: FlowRouter.url(attachment.url()), - file: filebase64, - name: attachment.name, - type: attachment.type, - }; + } else if (!Meteor.settings.public.sandstorm) { + Authentication.checkUserId(req.userId); + user = Users.findOne({ + _id: req.userId, + isAdmin: true, }); - - // we also have to export some user data - as the other elements only - // include id but we have to be careful: - // 1- only exports users that are linked somehow to that board - // 2- do not export any sensitive information - const users = {}; - result.members.forEach(member => { - users[member.userId] = true; - }); - result.lists.forEach(list => { - users[list.userId] = true; - }); - result.cards.forEach(card => { - users[card.userId] = true; - if (card.members) { - card.members.forEach(memberId => { - users[memberId] = true; - }); - } - }); - result.comments.forEach(comment => { - users[comment.userId] = true; - }); - result.activities.forEach(activity => { - users[activity.userId] = true; - }); - result.checklists.forEach(checklist => { - users[checklist.userId] = true; - }); - const byUserIds = { - _id: { - $in: Object.getOwnPropertyNames(users), - }, - }; - // we use whitelist to be sure we do not expose inadvertently - // some secret fields that gets added to User later. - const userFields = { - fields: { - _id: 1, - username: 1, - 'profile.fullname': 1, - 'profile.initials': 1, - 'profile.avatarUrl': 1, - }, - }; - result.users = Users.find(byUserIds, userFields) - .fetch() - .map(user => { - // user avatar is stored as a relative url, we export absolute - if ((user.profile || {}).avatarUrl) { - user.profile.avatarUrl = FlowRouter.url(user.profile.avatarUrl); - } - return user; + } + const exporter = new Exporter(boardId); + if (exporter.canExport(user)) { + body = params.query.delimiter + ? exporter.buildCsv(params.query.delimiter) + : exporter.buildCsv(); + res.writeHead(200, { + 'Content-Length': body[0].length, + 'Content-Type': params.query.delimiter ? 'text/csv' : 'text/tsv', }); - return result; - } - - canExport(user) { - const board = Boards.findOne(this._boardId); - return board && board.isVisibleBy(user); - } + res.write(body[0]); + res.end(); + } else { + res.writeHead(403); + res.end('Permission Error'); + } + }); } diff --git a/models/trelloCreator.js b/models/trelloCreator.js index c4be140b..1c5bcd93 100644 --- a/models/trelloCreator.js +++ b/models/trelloCreator.js @@ -369,7 +369,6 @@ export class TrelloCreator { // so we make it server only, and let UI catch up once it is done, forget about latency comp. const self = this; if (Meteor.isServer) { - // FIXME: Change to new model file.attachData(att.url, function(error) { file.boardId = boardId; file.cardId = cardId; diff --git a/models/wekanCreator.js b/models/wekanCreator.js index c5591a0b..9914f817 100644 --- a/models/wekanCreator.js +++ b/models/wekanCreator.js @@ -415,7 +415,6 @@ export class WekanCreator { const self = this; if (Meteor.isServer) { if (att.url) { - // FIXME: Change to new file library file.attachData(att.url, function(error) { file.boardId = boardId; file.cardId = cardId; @@ -441,7 +440,6 @@ export class WekanCreator { } }); } else if (att.file) { - // FIXME: Change to new file library file.attachData( Buffer.from(att.file, 'base64'), { diff --git a/sandstorm-pkgdef.capnp b/sandstorm-pkgdef.capnp index a902cb47..3dcf9e5f 100644 --- a/sandstorm-pkgdef.capnp +++ b/sandstorm-pkgdef.capnp @@ -22,10 +22,10 @@ const pkgdef :Spk.PackageDefinition = ( appTitle = (defaultText = "Wekan"), # The name of the app as it is displayed to the user. - appVersion = 404, + appVersion = 403, # Increment this for every release. - appMarketingVersion = (defaultText = "4.04.0~2020-05-24"), + appMarketingVersion = (defaultText = "4.03.0~2020-05-16"), # Human-readable presentation of the app version. minUpgradableAppVersion = 0, @@ -261,7 +261,6 @@ const myCommand :Spk.Manifest.Command = ( (key = "LDAP_ENABLE", value="false"), (key = "PASSWORD_LOGIN_ENABLED", value="true"), (key = "SANDSTORM", value="1"), - (key = "METEOR_SETTINGS", value = "{\"public\": {\"sandstorm\": true}}"), - (key = "ATTACHMENTS_STORE_PATH", value = "/var/attachments/") + (key = "METEOR_SETTINGS", value = "{\"public\": {\"sandstorm\": true}}") ] ); diff --git a/server/migrations.js b/server/migrations.js index 72b39ea7..5655bd1d 100644 --- a/server/migrations.js +++ b/server/migrations.js @@ -80,7 +80,7 @@ Migrations.add('lowercase-board-permission', () => { Migrations.add('change-attachments-type-for-non-images', () => { const newTypeForNonImage = 'application/octet-stream'; Attachments.find().forEach(file => { - if (!file.isImage) { + if (!file.isImage()) { Attachments.update( file._id, { @@ -1044,46 +1044,3 @@ Migrations.add('add-sort-field-to-boards', () => { } }); }); - -import { MongoInternals } from 'meteor/mongo'; - -Migrations.add('change-attachment-library', () => { - const fs = require('fs'); - CFSAttachments.find().forEach(file => { - const bucket = new MongoInternals.NpmModule.GridFSBucket(MongoInternals.defaultRemoteCollectionDriver().mongo.db, {bucketName: 'cfs_gridfs.attachments'}); - const gfsId = new MongoInternals.NpmModule.ObjectID(file.copies.attachments.key); - const reader = bucket.openDownloadStream(gfsId); - let store = Attachments.storagePath(); - if (store.charAt(store.length - 1) === '/') { - store = store.substring(0, store.length - 1); - } - const path = `${store}/${file.name()}`; - const fd = fs.createWriteStream(path); - reader.pipe(fd); - reader.on('end', () => { - let opts = { - fileName: file.name(), - type: file.type(), - size: file.size(), - fileId: file._id, - meta: { - userId: file.userId, - boardId: file.boardId, - cardId: file.cardId - } - }; - if (file.listId) { - opts.meta.listId = file.listId; - } - if (file.swimlaneId) { - opts.meta.swimlaneId = file.swimlaneId; - } - Attachments.addFile(path, opts, (err, fileRef) => { - if (err) { - console.log('error when migrating', file.name(), err); - } - }); - }); - }); -}); - diff --git a/server/old-attachments-migration.js b/server/old-attachments-migration.js deleted file mode 100644 index 3a6aa85d..00000000 --- a/server/old-attachments-migration.js +++ /dev/null @@ -1,212 +0,0 @@ -const localFSStore = process.env.ATTACHMENTS_STORE_PATH; -const storeName = 'attachments'; -const defaultStoreOptions = { - beforeWrite: fileObj => { - if (!fileObj.isImage()) { - return { - type: 'application/octet-stream', - }; - } - return {}; - }, -}; -let store; -if (localFSStore) { - // have to reinvent methods from FS.Store.GridFS and FS.Store.FileSystem - const fs = Npm.require('fs'); - const path = Npm.require('path'); - const mongodb = Npm.require('mongodb'); - const Grid = Npm.require('gridfs-stream'); - // calulate the absolute path here, because FS.Store.FileSystem didn't expose the aboslutepath or FS.Store didn't expose api calls :( - let pathname = localFSStore; - /*eslint camelcase: ["error", {allow: ["__meteor_bootstrap__"]}] */ - - if (!pathname && __meteor_bootstrap__ && __meteor_bootstrap__.serverDir) { - pathname = path.join( - __meteor_bootstrap__.serverDir, - `../../../cfs/files/${storeName}`, - ); - } - - if (!pathname) - throw new Error('FS.Store.FileSystem unable to determine path'); - - // Check if we have '~/foo/bar' - if (pathname.split(path.sep)[0] === '~') { - const homepath = - process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE; - if (homepath) { - pathname = pathname.replace('~', homepath); - } else { - throw new Error('FS.Store.FileSystem unable to resolve "~" in path'); - } - } - - // Set absolute path - const absolutePath = path.resolve(pathname); - - const _FStore = new FS.Store.FileSystem(storeName, { - path: localFSStore, - ...defaultStoreOptions, - }); - const GStore = { - fileKey(fileObj) { - const key = { - _id: null, - filename: null, - }; - - // If we're passed a fileObj, we retrieve the _id and filename from it. - if (fileObj) { - const info = fileObj._getInfo(storeName, { - updateFileRecordFirst: false, - }); - key._id = info.key || null; - key.filename = - info.name || - fileObj.name({ updateFileRecordFirst: false }) || - `${fileObj.collectionName}-${fileObj._id}`; - } - - // If key._id is null at this point, createWriteStream will let GridFS generate a new ID - return key; - }, - db: undefined, - mongoOptions: { useNewUrlParser: true }, - mongoUrl: process.env.MONGO_URL, - init() { - this._init(err => { - this.inited = !err; - }); - }, - _init(callback) { - const self = this; - mongodb.MongoClient.connect(self.mongoUrl, self.mongoOptions, function( - err, - db, - ) { - if (err) { - return callback(err); - } - self.db = db; - return callback(null); - }); - return; - }, - createReadStream(fileKey, options) { - const self = this; - if (!self.inited) { - self.init(); - return undefined; - } - options = options || {}; - - // Init GridFS - const gfs = new Grid(self.db, mongodb); - - // Set the default streamning settings - const settings = { - _id: new mongodb.ObjectID(fileKey._id), - root: `cfs_gridfs.${storeName}`, - }; - - // Check if this should be a partial read - if ( - typeof options.start !== 'undefined' && - typeof options.end !== 'undefined' - ) { - // Add partial info - settings.range = { - startPos: options.start, - endPos: options.end, - }; - } - return gfs.createReadStream(settings); - }, - }; - GStore.init(); - const CRS = 'createReadStream'; - const _CRS = `_${CRS}`; - const FStore = _FStore._transform; - FStore[_CRS] = FStore[CRS].bind(FStore); - FStore[CRS] = function(fileObj, options) { - let stream; - try { - const localFile = path.join( - absolutePath, - FStore.storage.fileKey(fileObj), - ); - const state = fs.statSync(localFile); - if (state) { - stream = FStore[_CRS](fileObj, options); - } - } catch (e) { - // file is not there, try GridFS ? - stream = undefined; - } - if (stream) return stream; - else { - try { - const stream = GStore[CRS](GStore.fileKey(fileObj), options); - return stream; - } catch (e) { - return undefined; - } - } - }.bind(FStore); - store = _FStore; -} else { - store = new FS.Store.GridFS(localFSStore ? `G${storeName}` : storeName, { - // XXX Add a new store for cover thumbnails so we don't load big images in - // the general board view - // If the uploaded document is not an image we need to enforce browser - // download instead of execution. This is particularly important for HTML - // files that the browser will just execute if we don't serve them with the - // appropriate `application/octet-stream` MIME header which can lead to user - // data leaks. I imagine other formats (like PDF) can also be attack vectors. - // See https://github.com/wekan/wekan/issues/99 - // XXX Should we use `beforeWrite` option of CollectionFS instead of - // collection-hooks? - // We should use `beforeWrite`. - ...defaultStoreOptions, - }); -} -CFSAttachments = new FS.Collection('attachments', { - stores: [store], -}); - -if (Meteor.isServer) { - Meteor.startup(() => { - CFSAttachments.files._ensureIndex({ cardId: 1 }); - }); - - CFSAttachments.allow({ - insert(userId, doc) { - return allowIsBoardMember(userId, Boards.findOne(doc.boardId)); - }, - update(userId, doc) { - return allowIsBoardMember(userId, Boards.findOne(doc.boardId)); - }, - remove(userId, doc) { - return allowIsBoardMember(userId, Boards.findOne(doc.boardId)); - }, - // We authorize the attachment download either: - // - if the board is public, everyone (even unconnected) can download it - // - if the board is private, only board members can download it - download(userId, doc) { - if (Meteor.isServer) { - return true; - } - const board = Boards.findOne(doc.boardId); - if (board.isPublic()) { - return true; - } else { - return board.hasMember(userId); - } - }, - - fetch: ['boardId'], - }); -} - -export default CFSAttachments; diff --git a/server/publications/boards.js b/server/publications/boards.js index ae82d28c..b80a6b23 100644 --- a/server/publications/boards.js +++ b/server/publications/boards.js @@ -131,7 +131,7 @@ Meteor.publishRelations('board', function(boardId, isArchived) { // Gather queries and send in bulk const cardComments = this.join(CardComments); cardComments.selector = _ids => ({ cardId: _ids }); - const attachments = this.join(Attachments.collection); + const attachments = this.join(Attachments); attachments.selector = _ids => ({ cardId: _ids }); const checklists = this.join(Checklists); checklists.selector = _ids => ({ cardId: _ids }); diff --git a/snap-src/bin/config b/snap-src/bin/config index ec0d3a0b..065b8685 100755 --- a/snap-src/bin/config +++ b/snap-src/bin/config @@ -93,9 +93,11 @@ DEFAULT_ACCOUNTS_LOCKOUT_UNKNOWN_USERS_FAILURE_WINDOW="15" KEY_ACCOUNTS_LOCKOUT_UNKNOWN_USERS_FAILURE_WINDOW="accounts-lockout-unknown-users-failure-window" DESCRIPTION_ATTACHMENTS_STORE_PATH="Allow wekan ower to specify where uploaded files to store on the server instead of the mongodb" -DEFAULT_ATTACHMENTS_STORE_PATH="/var/snap/wekan/common/uploads/" +DEFAULT_ATTACHMENTS_STORE_PATH="" KEY_ATTACHMENTS_STORE_PATH="attachments-store-path" +# Example, not in use: /var/snap/wekan/common/uploads/ + DESCRIPTION_MAX_IMAGE_PIXEL="Max image pixel: Allow to shrink attached/pasted image https://github.com/wekan/wekan/pull/2544" DEFAULT_MAX_IMAGE_PIXEL="" KEY_MAX_IMAGE_PIXEL="max-image-pixel" diff --git a/snap-src/bin/mongodb-control b/snap-src/bin/mongodb-control index 558aecd0..a78f7dfc 100755 --- a/snap-src/bin/mongodb-control +++ b/snap-src/bin/mongodb-control @@ -24,11 +24,11 @@ if test -f "$SNAP_COMMON/mongodb.log"; then rm -f "$SNAP_COMMON/mongodb.log" fi -# If uploads directory does not exist, create it. +# Not in use. If uploads directory does not exist, create it. # Wekan will store attachments there. -if [ ! -d "$SNAP_COMMON/uploads" ]; then - mkdir "$SNAP_COMMON/uploads" -fi +#if [ ! -d "$SNAP_COMMON/uploads" ]; then +# mkdir "$SNAP_COMMON/uploads" +#fi # Alternative: When starting MongoDB, and using logfile, truncate log to last 1000 lines of text. # 1) If file exists: |