handle empty fileInfo - don't switch to directory discovery

Summary:
handle empty fileInfo - don't switch to directory discovery

new option on WdtTransferRequest to force using fileInfo even if it is empty

Reviewed By: uddipta

Differential Revision: D2908775

fb-gh-sync-id: 387f39ab84cca1bd5bedd8e73d22060eaf4ce2da
shipit-source-id: 387f39ab84cca1bd5bedd8e73d22060eaf4ce2da

Sender.cpp

@@ -68,7 +68,8 @@ Sender::Sender(const std::string &destHost, const std::string &srcDir)
 // TODO: argghhhh
 Sender::Sender(const WdtTransferRequest &transferRequest)
     : Sender(transferRequest.hostName, transferRequest.directory,
-             transferRequest.ports, transferRequest.fileInfo) {
+             transferRequest.ports, transferRequest.fileInfo,
+             transferRequest.disableDirectoryTraversal) {
   transferRequest_ = transferRequest;
   if (getTransferId().empty()) {
     LOG(WARNING) << "Sender without transferId... will likely fail to connect";
@@ -81,11 +82,14 @@ Sender::Sender(const WdtTransferRequest &transferRequest)
 
 Sender::Sender(const std::string &destHost, const std::string &srcDir,
                const std::vector<int32_t> &ports,
-               const std::vector<WdtFileInfo> &srcFileInfo)
+               const std::vector<WdtFileInfo> &srcFileInfo,
+               bool disableDirectoryTraversal)
     : Sender(destHost, srcDir) {
   // TODO let's not copy vectors around to ourselves
   transferRequest_.ports = ports;
-  dirQueue_->setFileInfo(srcFileInfo);
+  if (!srcFileInfo.empty() || disableDirectoryTraversal) {
+    dirQueue_->setFileInfo(srcFileInfo);
+  }
   transferHistoryController_ =
       folly::make_unique<TransferHistoryController>(*dirQueue_);
 }

Sender.h

@@ -43,15 +43,18 @@ class Sender : public WdtBase {
    */
   Sender(const std::string &destHost, const std::string &srcDir);
 
+  // TODO: get rid of this constructor
   /**
    * @param destHost    destination hostname
    * @param srcDir      source directory
    * @param ports       list of destination ports
    * @param srcFileInfo list of (file, size) pair
+   * @param disableDirectoryTraversal use fileInfo even if empty
    */
   Sender(const std::string &destHost, const std::string &srcDir,
          const std::vector<int32_t> &ports,
-         const std::vector<WdtFileInfo> &srcFileInfo);
+         const std::vector<WdtFileInfo> &srcFileInfo,
+         bool disableDirectoryTraversal);
 
   /// Setup before start (@see WdtBase.h)
   const WdtTransferRequest &init() override;

WdtTransferRequest.h

@@ -159,6 +159,9 @@ struct WdtTransferRequest {
   /// Only used for the sender and when not using directory discovery
   std::vector<WdtFileInfo> fileInfo;
 
+  /// Use fileInfo even if empty (don't use the directory exploring)
+  bool disableDirectoryTraversal{false};
+
   /// Any error associated with this transfer request upon processing
   ErrorCode errorCode{OK};
 

test/wdt_stdin_test.sh

@@ -20,6 +20,20 @@ TEST_COUNT=0
 checkLastCmdStatus
 verifyTransferAndCleanup
 
+TEST_COUNT=1
+
+( $WDT_RECEIVER $WDTBIN_OPTS -directory $DIR/dst${TEST_COUNT} \
+    2> "$DIR/server${TEST_COUNT}.log" ; cd $SRC_DIR ) \
+ | $WDT_SENDER $WDTBIN_OPTS -directory $SRC_DIR -manifest - - \
+    2>&1 | tee "$DIR/client${TEST_COUNT}.log"
+
+checkLastCmdStatus
+
+if [ $(ls -1 $DIR/dst${TEST_COUNT} | wc -l) -ne 0 ] ; then
+  echo "This shouldnt have transferred any files (empty fileInfo)"
+  wdtExit 1
+fi
+
 echo "Test passed, deleting directory $DIR"
 rm -rf "$DIR"
 wdtExit 0

util/DirectorySourceQueue.cpp

@@ -85,6 +85,7 @@ void DirectorySourceQueue::setBlockSizeMbytes(int64_t blockSizeMbytes) {
 void DirectorySourceQueue::setFileInfo(
     const std::vector<WdtFileInfo> &fileInfo) {
   fileInfo_ = fileInfo;
+  exploreDirectory_ = false;
 }
 
 const std::vector<WdtFileInfo> &DirectorySourceQueue::getFileInfo() const {
@@ -191,12 +192,12 @@ bool DirectorySourceQueue::buildQueueSynchronously() {
   bool res = false;
   // either traverse directory or we already have a fixed set of candidate
   // files
-  if (!fileInfo_.empty()) {
+  if (exploreDirectory_) {
+    res = explore();
+  } else {
     LOG(INFO) << "Using list of file info. Number of files "
               << fileInfo_.size();
     res = enqueueFiles();
-  } else {
-    res = explore();
   }
   {
     std::lock_guard<std::mutex> lock(mutex_);

util/DirectorySourceQueue.h

@@ -386,6 +386,8 @@ class DirectorySourceQueue : public SourceQueue {
   int64_t numFilesOpenedWithDirect_{0};
   // Number of consumer threads (to tell between notify/notifyall)
   int64_t numClientThreads_{1};
+  // Should we explore or use fileInfo
+  bool exploreDirectory_{true};
 };
 }
 }

wdtCmdLine.cpp

@@ -147,6 +147,7 @@ void readManifest(std::istream &fin, WdtTransferRequest &req, bool dfltDirect) {
     bool odirect = fields.size() > 2 ? folly::to<bool>(fields[2]) : dfltDirect;
     req.fileInfo.emplace_back(fields[0], filesize, odirect);
   }
+  req.disableDirectoryTraversal = true;
 }
 
 namespace google {