QFileWriter.cc

Go to the documentation of this file.

#include "QFileWriter.hh"
#include "QFileList.hh"
#include "QRunDataHandle.hh"
#include "QGlobalWriterDispatcher.hh"
#include "QEventInspector.hh"
#include "QHeader.hh"
#include "TString.h"
#include "QPulseInfo.hh"
#include "QStringHandler.hh"
#include <TPRegexp.h>
#include <TObjArray.h>
#include <TObjString.h>
#include "QString.hh"
 
#include <fstream>
#include <unistd.h>
#include <sys/stat.h>
 
using namespace std;
 
Q_BEGIN_NAMESPACE
 
std::list<std::string> QFileWriter::fFullPathFilesList;
//_____________________________________________________________________________
QFileWriter::QFileWriter( const std::string& extension) : QWriter() {
    fFileExtension = extension;
    fWarnRunData = true;
    fOneFileOpenOnly = true;
    fDoGetTower = false;
}
//_____________________________________________________________________________
QFileWriter::~QFileWriter(){}
//_____________________________________________________________________________
void QFileWriter::Init(QEventInspector& evi) {
  // Where to write
  fOutputDir = GetString("OutputDir",".");
 
  fFileNamePrefix = GetString("OutputFilePrefix","QRFWriter_DEFAULT");
  fFileIdentifier = GetString("FileIdentifier","SyncWithRun",true);
  if(fFileIdentifier == "SyncWithTowerRun")
    fDoGetTower = true;
 
  // Write a file list
  fWriteFilesList = GetBool("WriteFilesList",true,false);
  if(fWriteFilesList) {
    fFilesListName = GetString("OutputFilesList","SyncWithRun",true);
    if(fFilesListName == "SyncWithTowerRun")
      fDoGetTower = true;
    fCurrFilesListName = fOutputDir + "/" + fFilesListName;
  }
 
 
  // If running multiple iterations, the input to the reader is the output
  // of this writer
  fSetListForReader = GetBool("SetListForReader",true,false);
  // If running multiple iterations, save the intermediary files.
  fSaveTempFiles = GetBool("SaveTempFiles",false,false);
  if(!fSaveTempFiles)
    fNewPartialOnRerun = GetBool("NewPartialOnRerun",false,false);
  else 
    fNewPartialOnRerun = false;
  // Store the framework config
  fKeepFrameWorkConfig = GetBool("KeepFrameWorkConfig",false,false);
 
  // Open a new file for each run
  fSyncWithRun = GetBool("SyncWithRun",true, false);
 
  // Open a new partial each time the reader does
  fSyncWithReader = GetBool("SyncWithReader",true,false);
  if(!fSyncWithReader ) {
    // If not syncing with reader, find out how many to write before
    // starting a new partial
    fPartialSize = GetInt("PartialSize",0);
    if(fPartialSize < 0)
      Panic("PartialSize cannot be < 0 (Read %d)",fPartialSize);
  }
 
  // If source of the event is cached, then use it. Otherwise it will
  // fall back to fRWCommon->fCurrentReaderFileName
  fUseEventCachedFileName =
    evi.GetEvent().Contains<QString>(fRWCommon->fReaderModuleName.c_str(),
                                     "OriginFileName");
 
  fEventNumber = 0;
  if(GetIteration() > 1 && fNewPartialOnRerun) return;
  fFilesList.clear();
  fFullPathFilesList.clear();
  fCurrentRun  = -999999;
  fPartial = 0;
 
  // set this writer for global writers
  std::stringstream currentWriterForGlobal;
  currentWriterForGlobal<<Q_CURRENT_WRITER<<fFileExtension;
  QGlobalWriterDispatcher::GetInstance().
    SetCurrentWriterFilename(currentWriterForGlobal.str());
  Require(evi);
  if(fDoGetTower)
    evi.GetEvent().Require<QPulseInfo>("DAQ","PulseInfo");
}
//_____________________________________________________________________________
void QFileWriter::Do(QEventInspector& evi) {
 
  
  if(IsToOpen(evi.GetEvent())) {
    // MV FIXME use event objects set by instead of RunData
    ReadHandle<QHeader> hh("Header");
    evi.GetEvent().Get("DAQ",hh);
    int run = 0;
    if(hh.IsValid()) run = hh.Get().GetRun();
    else Warn("No Header found, run number in filenames will be \"0\"");
    // Get the run type
    QRunDataHandle rHandle(run);
    GlobalData().Get<QRunData>("",&rHandle,"");
//    char runType = 'U';
 
    if(rHandle.IsValid()) {
      const QRunData& runData = rHandle.Get();
      run = runData.fNumber;
  //    runType = runData.fType.AsString().at(0);
    } else if(fWarnRunData) { 
      Warn("No RunData found, run type in filenames will be \"Unknown\"");
      Warn("No RunData found, tower in filenames will be 0");
      fWarnRunData = false;
    }
    
    // If we are on a new run, reset the partial    
    if(fSyncWithRun && run != fCurrentRun){
      fPartial = 0;
      fPartialsByFileId.clear();
    }
    fCurrentRun = run;
 
    // Close the previous files
    if(fFilesList.size() > 0 && fOneFileOpenOnly ) { 
      Debug("Closing %s",fFullPathFilesList.back().c_str());
      Close();
    }
 
    // Build the new file names
    BuildFileName(evi.GetEvent());
 
    // Write the file list (why do we write it here, before adding the
    // new file?)
    if(fWriteFilesList)
      WriteFilesList();
    //    fFileName = fFileNamePrefix +"_" + fFileId + fFileExtension;
    fFileFullPath = fOutputDir + "/" + fFileName;
    
    // Check that there is a FrameWorkConfig before overwriting
    if(fKeepFrameWorkConfig) {
      QGlobalDataManager dm;
      GlobalHandle<QFrameWorkConfig> fwcHandle("Config");
      dm.Get("Diana",&fwcHandle,fFileFullPath);
      if(fwcHandle.IsValid()) {
        Debug("Found valid FrameworkConfig");
        fRWCommon->fFrameWorkConfigOld = fwcHandle.Get();
      }
    }
    // Open the new file
    Open(fFileFullPath, evi);
    Info("Current file is %s",fFileFullPath.c_str());
    fFilesList.push_back(fFileName);
    fFullPathFilesList.push_back(fFileFullPath);
  }
  // Write to the tree
  Dump(evi);
  fEventNumber++;
}
//_____________________________________________________________________________
void QFileWriter::Done() {
  if(fFilesList.size() > 0 ) { 
    Debug("Closing %s",fFullPathFilesList.back().c_str());      
    Close();
  }
  Debug("Number of events written %d",fEventNumber);
  std::list<std::string>::iterator filesListIter;
  if(GetRunAgain() && fNewPartialOnRerun) return;
  
  // Run again on sequence needs files for temporary events so we
  // rename the existing ones
  if(GetRunAgain()){
    char iterBuf[4], pidBuf[32];
    snprintf(iterBuf,4,"%.2d",GetIteration());
    snprintf(pidBuf,32,"%d",getpid());
    for(filesListIter = fFilesList.begin();
        filesListIter != fFilesList.end(); filesListIter++) {
      std::string fileName = *filesListIter; 
      std::string tempFileName = std::string(iterBuf) + "_" + *filesListIter ;
      // moving output files to temp files
      rename((fOutputDir + "/" + fileName).c_str(),
             (fOutputDir + "/" + tempFileName).c_str());
      *filesListIter = tempFileName;
    }
    if(fWriteFilesList) remove(fCurrFilesListName.c_str());
    fCurrFilesListName = fOutputDir + "/" + std::string(pidBuf) + "_" 
      + std::string(iterBuf) + "_" 
      + fFileNamePrefix  +  ".list";
    if(fSetListForReader) 
      fRWCommon->fTempOutputFileList=fCurrFilesListName;
  }
  
  // write list file
  if(GetRunAgain() || fWriteFilesList)
    WriteFilesList();
  
  // remove files of previous iterations if not requested
  if(GetIteration() > 1 && !fSaveTempFiles) {
    for(filesListIter = fOldFilesList.begin();
        filesListIter != fOldFilesList.end(); filesListIter++) {
      std::string toBeRemoved = fOutputDir + "/" + *filesListIter;
      Debug("Removing %s", toBeRemoved.c_str()); 
      remove(toBeRemoved.c_str()); 
    }
    remove(fOldFilesListName.c_str());
    Debug("Removing %s", fOldFilesListName.c_str()); 
  }
  
  fOldFilesList = fFilesList;
  fOldFilesListName = fCurrFilesListName; 
}
//_____________________________________________________________________________
void QFileWriter::WriteFilesList() {
  QFileList::Write(fCurrFilesListName,fFilesList,fOutputDir);
  int FilePermissions = strtol(GetString("FilePermissions","0644",false).c_str(),NULL,8);
  chmod(fCurrFilesListName.c_str(),FilePermissions);
}
//_____________________________________________________________________________
void QFileWriter::RemoveFilesList() {
  if(fWriteFilesList) remove(fCurrFilesListName.c_str());
}
//_____________________________________________________________________________
bool QFileWriter::IsToOpen(const QEvent& ev) const {
  // MV FIXME use event objects set by instead of RunData
  ReadHandle<QHeader> hh("Header");
  ev.Get("DAQ",hh);
  
  int run = 0;
  if(hh.IsValid()) run = hh.Get().GetRun();
  else Warn("No Header found, run number in filenames will be \"0\"");
  
  bool retVal = false;
  if(fSyncWithReader) {
    if(fRWCommon->fReaderFileNumber > (int)fFilesList.size() ) {
      retVal = true;
    }
  }
  else if (fPartialSize == 0) {
    if(fCurrentRun < 0)
      retVal = true; 
    else
      retVal = false;
  }
  else if (fEventNumber % fPartialSize == 0)
    retVal = true;
  if(fSyncWithRun && run != fCurrentRun) 
    retVal = true;
  return retVal;
}
//_____________________________________________________________________________
const std::string &QFileWriter::GetCorrespondingReaderFileName(const QEvent &ev) const {
  std::string &readerFileName = fRWCommon->fReaderCurrentFileName;
  if(fUseEventCachedFileName)
    readerFileName = 
      ev.Get<Diana::QString>(fRWCommon->fReaderModuleName.c_str(),"OriginFileName");
  return readerFileName;
}
//_____________________________________________________________________________
void QFileWriter::BuildFileName(const QEvent &ev) {
 
  QRunDataHandle rHandle(fCurrentRun);
  GlobalData().Get<QRunData>("",&rHandle,"");
  char runType = 'U';
  int tower = 0;
  if(rHandle.IsValid()) {
    const QRunData& runData = rHandle.Get();
    runType = runData.fType.AsString().at(0);
  } else if(fWarnRunData) { 
    Warn("No RunData found, run type in filenames will be \"Unknown\"");
    Warn("No RunData found, tower in filenames will be 0");
    fWarnRunData = false;
  }
 
  if(fDoGetTower) {
    ReadHandle<QPulseInfo> ph("PulseInfo");
    ev.Get("DAQ",ph);
    /*
    int channel = Q_INT_DEFAULT;
    if(ph.IsValid()) channel = ph.Get().GetChannelId();
    QDetChannelCollectionHandle dccHandle;
    dccHandle.SetRun(fCurrentRun);
    GlobalData().Get("",&dccHandle,"");
    if(dccHandle.IsValid()) {
      const QDetChannel &detChannel = dccHandle.Get().Get(channel);
      tower = detChannel.fTd.fTdTower;
    }
    else
    */
    if(fWarnRunData) {
   
      Warn("No RunData found, tower in filenames will be 0");
    }
  }
  Bool_t usingPartial=true;
  if(fFileIdentifier == "SyncWithRun" || fFileIdentifier=="") {
    fFileId = Form("%06d_%c",fCurrentRun,runType);
  }
  else if(fFileIdentifier == "SyncWithTowerRun") {
    fFileId = Form("%06d_%03d_%c",fCurrentRun,tower,runType);
  }
  else if(fFileIdentifier == "Random"){
    fFileId = QStringHandler::RandomString(12);
    usingPartial = false;
  }
  else if(fFileIdentifier == "SyncWithReader") {
    static TPRegexp Re("_([0-9]+(_[0-9]+)?_[A-Z]_p[0-9]{3})");
    TObjArray *matches = Re.MatchS(fRWCommon->fReaderCurrentFileName);
    if(matches->GetEntries()>0){
      fFileId = ((TObjString *)matches->At(1))->GetString().Data();
    }
    else {
      fFileId = QStringHandler::RandomString(12);
      Warn("Could not parse reader file name for file ID! Will use random string"
           " identifier %s -> %s.",
           fRWCommon->fReaderCurrentFileName.c_str(),fFileId.c_str());
    }
    delete matches;
    usingPartial=false;
  }
  else if (fFileIdentifier != "") {
    fFileId = fFileIdentifier;
  }
  else 
    Panic("No valid file identifier provided! Output files will not be uniquely named!");
 
  if(usingPartial){
    fFileId+=Form("_p%03d",IncrementPartial(fFileId));
  }
  fFileName = fFileNamePrefix+"_"+fFileId+fFileExtension;
  
  if(fFilesListName == "SyncWithRun" || fFilesListName=="") {
    fCurrFilesListName =
      Form("%s/%s_%06d_%c.list",fOutputDir.c_str(),fFileNamePrefix.c_str(),
           fCurrentRun,runType);
  }
  else if(fFilesListName == "SyncWithTowerRun") {
    fCurrFilesListName =
      Form("%s/%s_%06d_%03d_%c.list",fOutputDir.c_str(),fFileNamePrefix.c_str(),
           fCurrentRun,tower,runType);
  }
}
//_____________________________________________________________________________
int QFileWriter::IncrementPartial(const std::string &fileId) {
  if(!fPartialsByFileId.count(fileId))
    fPartialsByFileId[fileId]=0;
  int &partial=fPartialsByFileId[fileId];
  return ++partial;
}
 
Q_END_NAMESPACE