144ebf8b0331cb3a994a8df16fb63fdcdbc37d927Nick Lewycky//===--- llvm/Support/DataStream.cpp - Lazy streamed data -----------------===// 22ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// 32ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// The LLVM Compiler Infrastructure 42ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// 52ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// This file is distributed under the University of Illinois Open Source 62ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// License. See LICENSE.TXT for details. 72ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// 82ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff//===----------------------------------------------------------------------===// 92ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// 102ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// This file implements DataStreamer, which fetches bytes of Data from 112ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// a stream source. It provides support for streaming (lazy reading) of 122ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// bitcode. An example implementation of streaming from a file or stdin 132ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// is included. 142ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// 152ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff//===----------------------------------------------------------------------===// 162ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff 172ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff#include "llvm/Support/DataStream.h" 18d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "llvm/ADT/Statistic.h" 19c1b49b56d4132efa2e06deb8f23508d0de4c8800Rafael Espindola#include "llvm/Support/FileSystem.h" 20eb446511ce5f1761962f12f1e8a0c7b342733d1cDerek Schuff#include "llvm/Support/Program.h" 212ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff#include <cerrno> 222ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff#include <cstdio> 23d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include <string> 24cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines#include <system_error> 252ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff#if !defined(_MSC_VER) && !defined(__MINGW32__) 262ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff#include <unistd.h> 272ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff#else 282ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff#include <io.h> 292ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff#endif 302ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuffusing namespace llvm; 312ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff 32dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#define DEBUG_TYPE "Data-stream" 33dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 342ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// Interface goals: 352ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// * StreamableMemoryObject doesn't care about complexities like using 362ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// threads/async callbacks to actually overlap download+compile 372ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// * Don't want to duplicate Data in memory 382ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// * Don't need to know total Data len in advance 392ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// Non-goals: 402ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// StreamableMemoryObject already has random access so this interface only does 412ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// in-order streaming (no arbitrary seeking, else we'd have to buffer all the 422ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// Data here in addition to MemoryObject). This also means that if we want 432ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// to be able to to free Data, BitstreamBytes/BitcodeReader will implement it 442ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff 452ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek SchuffSTATISTIC(NumStreamFetches, "Number of calls to Data stream fetch"); 462ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff 472ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuffnamespace llvm { 482ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek SchuffDataStreamer::~DataStreamer() {} 492ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff} 502ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff 512ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuffnamespace { 522ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff 532ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// Very simple stream backed by a file. Mostly useful for stdin and debugging; 542ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff// actual file access is probably still best done with mmap. 552ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuffclass DataFileStreamer : public DataStreamer { 562ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff int Fd; 572ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuffpublic: 582ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff DataFileStreamer() : Fd(0) {} 592ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff virtual ~DataFileStreamer() { 602ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff close(Fd); 612ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff } 6236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines size_t GetBytes(unsigned char *buf, size_t len) override { 632ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff NumStreamFetches++; 642ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff return read(Fd, buf, len); 652ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff } 662ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff 67cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines std::error_code OpenFile(const std::string &Filename) { 68eb446511ce5f1761962f12f1e8a0c7b342733d1cDerek Schuff if (Filename == "-") { 692ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff Fd = 0; 709f1d9fd1964d82f3e801efb71518144492cdf290Rafael Espindola sys::ChangeStdinToBinary(); 71cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines return std::error_code(); 72eb446511ce5f1761962f12f1e8a0c7b342733d1cDerek Schuff } 73c1b49b56d4132efa2e06deb8f23508d0de4c8800Rafael Espindola 74c1b49b56d4132efa2e06deb8f23508d0de4c8800Rafael Espindola return sys::fs::openFileForRead(Filename, Fd); 752ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff } 762ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff}; 772ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff 782ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff} 792ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff 802ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuffnamespace llvm { 812ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek SchuffDataStreamer *getDataFileStreamer(const std::string &Filename, 822ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff std::string *StrError) { 832ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff DataFileStreamer *s = new DataFileStreamer(); 84cd81d94322a39503e4a3e87b6ee03d4fcb3465fbStephen Hines if (std::error_code e = s->OpenFile(Filename)) { 852ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff *StrError = std::string("Could not open ") + Filename + ": " + 862ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff e.message() + "\n"; 87dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return nullptr; 882ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff } 892ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff return s; 902ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff} 912ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff 922ea93875b2f2900b9d244dfd7649c9ed02a34cd7Derek Schuff} 93