Fixed upload streams

Lukas added a lot of pretty things to the upload waiting screen.
Current bug: progress bar is not 100% working.

app/classes/shared/helpers.py

@@ -578,7 +578,16 @@ class Helpers:
         return output
 
     @staticmethod
-    def in_path(x, y):
+    def in_path(parent_path, child_path):
+        # Smooth out relative path names, note: if you are concerned about symbolic links, you should use os.path.realpath too
+        parent_path = os.path.abspath(parent_path)
+        child_path = os.path.abspath(child_path)
+
+        # Compare the common path of the parent and child path with the common path of just the parent path. Using the commonpath method on just the parent path will regularise the path name in the same way as the comparison that deals with both paths, removing any trailing path separator
+        return os.path.commonpath([parent_path]) == os.path.commonpath([parent_path, child_path])
+
+    @staticmethod
+    def in_path_old(x, y):
         return os.path.abspath(y).__contains__(os.path.abspath(x))
     
     @staticmethod

app/classes/web/tornado.py

@@ -28,7 +28,6 @@ try:
     from app.classes.web.static_handler import CustomStaticHandler
     from app.classes.shared.translation import translation
     from app.classes.web.upload_handler import UploadHandler
-    from app.classes.web.upload_handler import ProxyHandler
 
 except ModuleNotFoundError as e:
     logger.critical("Import Error: Unable to load {} module".format(e, e.name))
@@ -131,8 +130,7 @@ class Webserver:
             (r'/api/stats/servers', ServersStats, handler_args),
             (r'/api/stats/node', NodeStats, handler_args),
             (r'/ws', SocketHandler, handler_args),
-            (r'/upload', UploadHandler, handler_args),
+            (r'/upload', UploadHandler),
-            (r'/proxy', ProxyHandler, handler_args)
             ]
 
         app = tornado.web.Application(

app/classes/web/upload_handler.py

@@ -1,88 +1,79 @@
-from tornado.concurrent import Future
+import tornado.options
-from tornado.escape import utf8
+import tornado.web
-from tornado import gen
+import tornado.httpserver
-from tornado.httpclient import AsyncHTTPClient
+from tornado.options import options
-from tornado.ioloop import IOLoop
+from app.classes.shared.models import db_helper, Enum_Permissions
-from tornado.options import parse_command_line, define, options
+from app.classes.shared.helpers import helper
-from tornado.web import Application, RequestHandler, stream_request_body
+from app.classes.web.websocket_helper import websocket_helper
-
+from app.classes.shared.console import console
 import logging
-import toro
+import os
+import json
+import time
 
 logger = logging.getLogger(__name__)
 
+# Class&Function Defination
+MAX_STREAMED_SIZE = 1024 * 1024 * 1024
 
-define('server_delay', default=2.0)
+@tornado.web.stream_request_body
-define('client_delay', default=1.0)
+class UploadHandler(tornado.web.RequestHandler):
-define('num_chunks', default=40)
-
-@stream_request_body
-class UploadHandler(RequestHandler):
     def prepare(self):
-        print("In PREPARE")
+        self.do_upload = True
-        logger.info('UploadHandler.prepare')
+        user_data = json.loads(self.get_secure_cookie('user_data'))
+        user_id = user_data['user_id']
 
-    @gen.coroutine
+        server_id = self.request.headers.get('X-ServerId', None)
-    def data_received(self, chunk):
-        print("In RECIEVED")
-        logger.info('UploadHandler.data_received(%d bytes: %r)',
-                     len(chunk), chunk[:9])
-        yield gen.Task(IOLoop.current().call_later, options.server_delay)
 
-    def put(self):
+        if user_id is None:
-        print("In PUT")
+            logger.warning('User ID not found in upload handler call')
-        logger.info('UploadHandler.put')
+            console.warning('User ID not found in upload handler call')
-        self.write('ok')
+            self.do_upload = False
         
-@stream_request_body
+        if server_id is None:
-class ProxyHandler(RequestHandler):
+            logger.warning('Server ID not found in upload handler call')
-    def prepare(self):
+            console.warning('Server ID not found in upload handler call')
-        logger.info('ProxyHandler.prepare')
+            self.do_upload = False
-        self.chunks = toro.Queue(1)
-        self.fetch_future = AsyncHTTPClient().fetch(
-            'http://localhost:%d/upload' % options.port,
-            method='PUT',
-            body_producer=self.body_producer,
-            request_timeout=3600.0)
 
-    @gen.coroutine
+        user_permissions = db_helper.get_user_permissions_list(user_id, server_id)
-    def body_producer(self, write):
+        if Enum_Permissions.Files not in user_permissions:
-        while True:
+            logger.warning(f'User {user_id} tried to upload a file to {server_id} without permissions!')
-            chunk = yield self.chunks.get()
+            console.warning(f'User {user_id} tried to upload a file to {server_id} without permissions!')
-            if chunk is None:
+            self.do_upload = False
-                return
-            yield write(chunk)
 
-    @gen.coroutine
+        path = self.request.headers.get('X-Path', None)
-    def data_received(self, chunk):
+        filename = self.request.headers.get('X-FileName', None)
-        logger.info('ProxyHandler.data_received(%d bytes: %r)',
+        full_path = os.path.join(path, filename)
-                     len(chunk), chunk[:9])
-        yield self.chunks.put(chunk)
 
-    @gen.coroutine
+        if not helper.in_path(db_helper.get_server_data_by_id(server_id)['path'], full_path):
-    def put(self):
+            print(user_id, server_id, db_helper.get_server_data_by_id(server_id)['path'], full_path)
-        logger.info('ProxyHandler.put')
+            logger.warning(f'User {user_id} tried to upload a file to {server_id} but the path is not inside of the server!')
-        # Write None to the chunk queue to signal body_producer to exit,
+            console.warning(f'User {user_id} tried to upload a file to {server_id} but the path is not inside of the server!')
-        # then wait for the request to finish.
+            self.do_upload = False
-        yield self.chunks.put(None)
-        response = yield self.fetch_future
-        self.set_status(response.code)
-        self.write(response.body)
         
-@gen.coroutine
+        if self.do_upload:
-def client():
+            try:
-    @gen.coroutine
+                self.f = open(full_path, "wb")
-    def body_producer(write):
+            except Exception as e:
-        for i in range(options.num_chunks):
+                logger.error("Upload failed with error: {}".format(e))
-            yield gen.Task(IOLoop.current().call_later, options.client_delay)
+                self.do_upload = False
-            chunk = ('chunk %02d ' % i) * 10000
+        # If max_body_size is not set, you cannot upload files > 100MB
-            logger.info('client writing %d bytes: %r', len(chunk), chunk[:9])
+        self.request.connection.set_max_body_size(MAX_STREAMED_SIZE)
-            yield write(utf8(chunk))
 
-    response = yield AsyncHTTPClient().fetch(
+    def post(self):
-        'http://localhost:%d/proxy' % options.port,
+        logger.info("Upload completed")
-        method='PUT',
+        
-        body_producer=body_producer,
+        
-        request_timeout=3600.0)
+        if self.do_upload:
-    logger.info('client finished with response %d: %r',
+            time.sleep(5)
-                 response.code, response.body)
+            websocket_helper.broadcast('close_upload_box', 'success')
+            self.finish('success') # Nope, I'm sending "success"
+            self.f.close()
+        else:
+            time.sleep(5)
+            websocket_helper.broadcast('close_upload_box', 'error')
+            self.finish('error')
+
+    def data_received(self, data):
+        if self.do_upload:
+            self.f.write(data)

app/frontend/templates/panel/server_files.html

@@ -321,6 +321,10 @@
         regex: /^properties$/,
         replaceWith: 'ace/mode/properties'
       },
+      {
+        regex: /^log$/,
+        replaceWith: 'ace/mode/txt'
+      },
     ];
 
     var filePath = '';
@@ -520,6 +524,42 @@
       });
       window.location.href = "/panel/server_detail?id={{ data['server_stats']['server_id']['server_id'] }}&subpage=files"
     }
+
+  function sendFile(file, path, server_id, onProgress){
+    var xmlHttpRequest = new XMLHttpRequest();
+    var token = getCookie("_xsrf")
+    var fileName = file.name
+    var target = '/upload?server_id=' + server_id
+    var mimeType = file.type
+
+    xmlHttpRequest.open('POST', target, true);
+    xmlHttpRequest.setRequestHeader('X-Content-Type', mimeType);
+    xmlHttpRequest.setRequestHeader('X-XSRFToken', token);
+    xmlHttpRequest.setRequestHeader('X-Content-Disposition', 'attachment; filename="' + fileName + '"');
+    xmlHttpRequest.setRequestHeader('X-Path', path);
+    xmlHttpRequest.setRequestHeader('X-FileName', fileName);
+    xmlHttpRequest.setRequestHeader('X-ServerId', "{{ data['server_stats']['server_id']['server_id'] }}");
+    xmlHttpRequest.upload.addEventListener('progress', (event) =>
+      onProgress(Math.floor(event.loaded / event.total * 100)), false);
+    xmlHttpRequest.addEventListener('load', (event) => {
+      if (event.target.responseText == 'success') {
+        console.log('Upload for file', file.name, 'was successful!')
+      }
+      else {
+        alert('Upload failed with response: ' + event.target.responseText);
+        doUpload = false;
+      }
+    }, false);
+    xmlHttpRequest.addEventListener('error', (e) => {
+      console.error('Error while uploading file', file.name + '.', 'Event:', e)
+    }, false);
+    
+    xmlHttpRequest.send(file);
+  }
+
+  var uploadWaitDialog;
+  var doUpload = true;
+
   function uploadFilesE(event){
     path = event.target.parentElement.getAttribute('data-path');
     console.log("PATH: " + path);
@@ -530,42 +570,69 @@
             "<input id='files' name='files' type='file' style='display:none;' multiple='true'>" +
             "<i class='fa fa-cloud-upload fa-3x'></i>" +
             "<br />" +
-          "Click Here To Upload" +
+            "Click Here To Select Files" +
         "</label></form>" +
         "<br />" +
         "<ul style='margin-left:5px !important;' id='fileList'></ul>" +
     "</div><div class='clearfix'></div>";
-
     bootbox.dialog({
         message: uploadHtml,
-      title: "File Upload",
+        title: "Upload Files to : "+path,
         buttons: {
             success: {
                 label: "Upload",
                 className: "btn-default",
-              callback: function () {
+                callback: async function () {
-                  $('#upload_file').submit(); //.trigger('submit');
+                    files = document.getElementById("files");
-                  files = document.getElementById("upload_files").files
+                    uploadWaitDialog = bootbox.dialog({
-                  for (i = 0; i < files.length; i++){
+                      message: `
-
+                        <p class="text-center mb-0">
-                  var xmlHttpRequest = new XMLHttpRequest();
+                          <i class="fa fa-spin fa-cog"></i>
-                  var file = files[i]
+                          Please wait while we upload your files... This may take a while.<br>
-                  var fileName = files[i].name
+                          <strong>DO NOT CLOSE THIS PAGE.</strong>
-                  var target = '/proxy'
+                        </p>
-                  var mimeType = files[i].type
+                        <div class="progress" id="upload-progress-bar-parent">
-
+                        </div>
-                  xmlHttpRequest.open('POSTT', target, true);
+                      `,
-                  console.log(xmlHttpRequest);
+                      closeButton: false
-                  xmlHttpRequest.setRequestHeader('Content-Type', mimeType);
+                  });
-                  xmlHttpRequest.setRequestHeader('Content-Disposition', 'attachment; filename="' + fileName + '"');
+                  let nFiles = files.files.length;
-                  xmlHttpRequest.send(file);
+                  for(i=0; i < files.files.length; i++) {
-
+                    if (!doUpload) {
+                      doUpload = true;
+                      hideUploadBox();
+                      break;
                     }
+                    console.log(files.files[i]);
+                    const progressHtml = `
+                    <div>
+                      ${path + '/' + files.files[i]}:
+                      <div
+                        id="upload-progress-bar-${i + 1}"
+                        class="progress-bar progress-bar-striped progress-bar-animated"
+                        role="progressbar"
+                        style="width: 0%"
+                        aria-valuenow="0"
+                        aria-valuemin="0"
+                        aria-valuemax="100"
+                      ></div>
+                    </div>
+                    `;
+                    $('#upload-progress-bar-parent').append(progressHtml);
+                    sendFile(files.files[i], path, server_id, (progress) => {
+                      $(`#upload-progress-bar-${i + 1}`).attr('aria-valuenow', progress)
+                      $(`#upload-progress-bar-${i + 1}`).css('width', progress + '%')
+                    });
+                  }
+                  hideUploadBox();
+                  //$('#upload_file').submit(); //.trigger('submit');
                 }
             }
         }
   });
 
+  
+
   var fileList = document.getElementById("files");
   fileList.addEventListener("change", function (e) {
       var list = "";
@@ -577,18 +644,6 @@
   }, false);
 });
     }
-
-
-    function uploadFiles(e){
-    path = event.target.parentElement.getAttribute('data-path');
-    server_id = {{ data['server_stats']['server_id']['server_id'] }};
-  var uploadHtml = '<form enctype="multipart/form-data" action="/ajax/upload_files?id=' + server_id +'&path='+ path +'"method="post">{% raw xsrf_form_html() %}<div class="form-group">'+"<label class='upload-area' style='width:100%;text-align:center;' for='fupload'>" +'<input id="files" type="file" name="files" multiple>' +"<i class='fa fa-cloud-upload fa-3x'></i>" +"<br />" +"Upload Files Here" +"</label>" +"<br />" +"<span style='margin-left:5px !important;' id='fileList'></span>"+'</div><br><br><input id="upload_file" type="submit"value="Upload File" class="btn btn-success hidden"/></form>';
-  bootbox.dialog({
-      message: uploadHtml,
-      title: "Upload Files To "+path,
-  });
-}
-
     function getTreeView() {
       $.ajax({
         type: "GET",
@@ -688,12 +743,25 @@
 
     document.addEventListener('click', function(e){
   let inside = (e.target.closest('#files-tree-nav'));
-  if(!inside){
   let contextMenu = document.getElementById('files-tree-nav');
+  if(!inside){
+    contextMenu.setAttribute('style', 'display:none');
+  }else{
     contextMenu.setAttribute('style', 'display:none');
   }
 });
 
+    if (webSocket) {
+      webSocket.on('close_upload_box', function (close_upload_box) {
+        hideUploadBox();
+      });
+    }
+
+    function hideUploadBox(){
+      if (!uploadWaitDialog) return;
+      uploadWaitDialog.modal('hide');
+      getTreeView();
+    }
 
     function createFileE(event) {
       bootbox.prompt("{% raw translate('serverFiles', 'createFileQuestion') %}", function(result) {

chat.txt

@@ -0,0 +1 @@
+Sure