final/MultiSource/Benchmarks/7zip/CPP/7zip/Archive/7z/7zIn.cpp

// 7zIn.cpp

#include "StdAfx.h"

#include "../../../../C/7zCrc.h"
#include "../../../../C/CpuArch.h"

#include "../../Common/StreamObjects.h"
#include "../../Common/StreamUtils.h"

#include "7zDecode.h"
#include "7zIn.h"

#define Get16(p) GetUi16(p)
#define Get32(p) GetUi32(p)
#define Get64(p) GetUi64(p)

// define FORMAT_7Z_RECOVERY if you want to recover multivolume archives with empty StartHeader
#ifndef _SFX
#define FORMAT_7Z_RECOVERY
#endif

namespace NArchive {
namespace N7z {

static void BoolVector_Fill_False(CBoolVector &v, int size)
{
  v.Clear();
  v.Reserve(size);
  for (int i = 0; i < size; i++)
    v.Add(false);
}

static bool BoolVector_GetAndSet(CBoolVector &v, UInt32 index)
{
  if (index >= (UInt32)v.Size())
    return true;
  bool res = v[index];
  v[index] = true;
  return res;
}

bool CFolder::CheckStructure() const
{
  const int kNumCodersMax = sizeof(UInt32) * 8; // don't change it
  const int kMaskSize = sizeof(UInt32) * 8; // it must be >= kNumCodersMax
  const int kNumBindsMax = 32;

  if (Coders.Size() > kNumCodersMax || BindPairs.Size() > kNumBindsMax)
    return false;

  {
    CBoolVector v;
    BoolVector_Fill_False(v, BindPairs.Size() + PackStreams.Size());
    
    int i;
    for (i = 0; i < BindPairs.Size(); i++)
      if (BoolVector_GetAndSet(v, BindPairs[i].InIndex))
        return false;
    for (i = 0; i < PackStreams.Size(); i++)
      if (BoolVector_GetAndSet(v, PackStreams[i]))
        return false;
    
    BoolVector_Fill_False(v, UnpackSizes.Size());
    for (i = 0; i < BindPairs.Size(); i++)
      if (BoolVector_GetAndSet(v, BindPairs[i].OutIndex))
        return false;
  }
  
  UInt32 mask[kMaskSize];
  int i;
  for (i = 0; i < kMaskSize; i++)
    mask[i] = 0;

  {
    CIntVector inStreamToCoder, outStreamToCoder;
    for (i = 0; i < Coders.Size(); i++)
    {
      CNum j;
      const CCoderInfo &coder = Coders[i];
      for (j = 0; j < coder.NumInStreams; j++)
        inStreamToCoder.Add(i);
      for (j = 0; j < coder.NumOutStreams; j++)
        outStreamToCoder.Add(i);
    }
    
    for (i = 0; i < BindPairs.Size(); i++)
    {
      const CBindPair &bp = BindPairs[i];
      mask[inStreamToCoder[bp.InIndex]] |= (1 << outStreamToCoder[bp.OutIndex]);
    }
  }
  
  for (i = 0; i < kMaskSize; i++)
    for (int j = 0; j < kMaskSize; j++)
      if (((1 << j) & mask[i]) != 0)
        mask[i] |= mask[j];

  for (i = 0; i < kMaskSize; i++)
    if (((1 << i) & mask[i]) != 0)
      return false;

  return true;
}

class CInArchiveException {};

static void ThrowException() { throw CInArchiveException(); }
static inline void ThrowEndOfData()   { ThrowException(); }
static inline void ThrowUnsupported() { ThrowException(); }
static inline void ThrowIncorrect()   { ThrowException(); }
static inline void ThrowUnsupportedVersion() { ThrowException(); }

/*
class CInArchiveException
{
public:
  enum CCauseType
  {
    kUnsupportedVersion = 0,
    kUnsupported,
    kIncorrect,
    kEndOfData
  } Cause;
  CInArchiveException(CCauseType cause): Cause(cause) {};
};

static void ThrowException(CInArchiveException::CCauseType c) { throw CInArchiveException(c); }
static void ThrowEndOfData()   { ThrowException(CInArchiveException::kEndOfData); }
static void ThrowUnsupported() { ThrowException(CInArchiveException::kUnsupported); }
static void ThrowIncorrect()   { ThrowException(CInArchiveException::kIncorrect); }
static void ThrowUnsupportedVersion() { ThrowException(CInArchiveException::kUnsupportedVersion); }
*/

class CStreamSwitch
{
  CInArchive *_archive;
  bool _needRemove;
public:
  CStreamSwitch(): _needRemove(false) {}
  ~CStreamSwitch() { Remove(); }
  void Remove();
  void Set(CInArchive *archive, const Byte *data, size_t size);
  void Set(CInArchive *archive, const CByteBuffer &byteBuffer);
  void Set(CInArchive *archive, const CObjectVector<CByteBuffer> *dataVector);
};

void CStreamSwitch::Remove()
{
  if (_needRemove)
  {
    _archive->DeleteByteStream();
    _needRemove = false;
  }
}

void CStreamSwitch::Set(CInArchive *archive, const Byte *data, size_t size)
{
  Remove();
  _archive = archive;
  _archive->AddByteStream(data, size);
  _needRemove = true;
}

void CStreamSwitch::Set(CInArchive *archive, const CByteBuffer &byteBuffer)
{
  Set(archive, byteBuffer, byteBuffer.GetCapacity());
}

void CStreamSwitch::Set(CInArchive *archive, const CObjectVector<CByteBuffer> *dataVector)
{
  Remove();
  Byte external = archive->ReadByte();
  if (external != 0)
  {
    int dataIndex = (int)archive->ReadNum();
    if (dataIndex < 0 || dataIndex >= dataVector->Size())
      ThrowIncorrect();
    Set(archive, (*dataVector)[dataIndex]);
  }
}

Byte CInByte2::ReadByte()
{
  if (_pos >= _size)
    ThrowEndOfData();
  return _buffer[_pos++];
}

void CInByte2::ReadBytes(Byte *data, size_t size)
{
  if (size > _size - _pos)
    ThrowEndOfData();
  for (size_t i = 0; i < size; i++)
    data[i] = _buffer[_pos++];
}

void CInByte2::SkipData(UInt64 size)
{
  if (size > _size - _pos)
    ThrowEndOfData();
  _pos += (size_t)size;
}

void CInByte2::SkipData()
{
  SkipData(ReadNumber());
}

UInt64 CInByte2::ReadNumber()
{
  if (_pos >= _size)
    ThrowEndOfData();
  Byte firstByte = _buffer[_pos++];
  Byte mask = 0x80;
  UInt64 value = 0;
  for (int i = 0; i < 8; i++)
  {
    if ((firstByte & mask) == 0)
    {
      UInt64 highPart = firstByte & (mask - 1);
      value += (highPart << (i * 8));
      return value;
    }
    if (_pos >= _size)
      ThrowEndOfData();
    value |= ((UInt64)_buffer[_pos++] << (8 * i));
    mask >>= 1;
  }
  return value;
}

CNum CInByte2::ReadNum()
{
  UInt64 value = ReadNumber();
  if (value > kNumMax)
    ThrowUnsupported();
  return (CNum)value;
}

UInt32 CInByte2::ReadUInt32()
{
  if (_pos + 4 > _size)
    ThrowEndOfData();
  UInt32 res = Get32(_buffer + _pos);
  _pos += 4;
  return res;
}

UInt64 CInByte2::ReadUInt64()
{
  if (_pos + 8 > _size)
    ThrowEndOfData();
  UInt64 res = Get64(_buffer + _pos);
  _pos += 8;
  return res;
}

void CInByte2::ReadString(UString &s)
{
  const Byte *buf = _buffer + _pos;
  size_t rem = (_size - _pos) / 2 * 2;
  {
    size_t i;
    for (i = 0; i < rem; i += 2)
      if (buf[i] == 0 && buf[i + 1] == 0)
        break;
    if (i == rem)
      ThrowEndOfData();
    rem = i;
  }
  int len = (int)(rem / 2);
  if (len < 0 || (size_t)len * 2 != rem)
    ThrowUnsupported();
  wchar_t *p = s.GetBuffer(len);
  int i;
  for (i = 0; i < len; i++, buf += 2)
    p[i] = (wchar_t)Get16(buf);
  s.ReleaseBuffer(len);
  _pos += rem + 2;
}

static inline bool TestSignature(const Byte *p)
{
  for (int i = 0; i < kSignatureSize; i++)
    if (p[i] != kSignature[i])
      return false;
  return CrcCalc(p + 12, 20) == GetUi32(p + 8);
}

#ifdef FORMAT_7Z_RECOVERY
static inline bool TestSignature2(const Byte *p)
{
  int i;
  for (i = 0; i < kSignatureSize; i++)
    if (p[i] != kSignature[i])
      return false;
  if (CrcCalc(p + 12, 20) == GetUi32(p + 8))
    return true;
  for (i = 8; i < kHeaderSize; i++)
    if (p[i] != 0)
      return false;
  return (p[6] != 0 || p[7] != 0);
}
#else
#define TestSignature2(p) TestSignature(p)
#endif

HRESULT CInArchive::FindAndReadSignature(IInStream *stream, const UInt64 *searchHeaderSizeLimit)
{
  RINOK(ReadStream_FALSE(stream, _header, kHeaderSize));

  if (TestSignature2(_header))
    return S_OK;

  CByteBuffer byteBuffer;
  const UInt32 kBufferSize = (1 << 16);
  byteBuffer.SetCapacity(kBufferSize);
  Byte *buffer = byteBuffer;
  UInt32 numPrevBytes = kHeaderSize;
  memcpy(buffer, _header, kHeaderSize);
  UInt64 curTestPos = _arhiveBeginStreamPosition;
  for (;;)
  {
    if (searchHeaderSizeLimit != NULL)
      if (curTestPos - _arhiveBeginStreamPosition > *searchHeaderSizeLimit)
        break;
    do
    {
      UInt32 numReadBytes = kBufferSize - numPrevBytes;
      UInt32 processedSize;
      RINOK(stream->Read(buffer + numPrevBytes, numReadBytes, &processedSize));
      numPrevBytes += processedSize;
      if (processedSize == 0)
        return S_FALSE;
    }
    while (numPrevBytes <= kHeaderSize);
    UInt32 numTests = numPrevBytes - kHeaderSize;
    for (UInt32 pos = 0; pos < numTests; pos++)
    {
      for (; buffer[pos] != '7' && pos < numTests; pos++);
      if (pos == numTests)
        break;
      if (TestSignature(buffer + pos))
      {
        memcpy(_header, buffer + pos, kHeaderSize);
        curTestPos += pos;
        _arhiveBeginStreamPosition = curTestPos;
        return stream->Seek(curTestPos + kHeaderSize, STREAM_SEEK_SET, NULL);
      }
    }
    curTestPos += numTests;
    numPrevBytes -= numTests;
    memmove(buffer, buffer + numTests, numPrevBytes);
  }
  return S_FALSE;
}

// S_FALSE means that file is not archive
HRESULT CInArchive::Open(IInStream *stream, const UInt64 *searchHeaderSizeLimit)
{
  HeadersSize = 0;
  Close();
  RINOK(stream->Seek(0, STREAM_SEEK_CUR, &_arhiveBeginStreamPosition))
  RINOK(FindAndReadSignature(stream, searchHeaderSizeLimit));
  _stream = stream;
  return S_OK;
}
  
void CInArchive::Close()
{
  _stream.Release();
}

void CInArchive::ReadArchiveProperties(CInArchiveInfo & /* archiveInfo */)
{
  for (;;)
  {
    if (ReadID() == NID::kEnd)
      break;
    SkipData();
  }
}

void CInArchive::GetNextFolderItem(CFolder &folder)
{
  CNum numCoders = ReadNum();

  folder.Coders.Clear();
  folder.Coders.Reserve((int)numCoders);
  CNum numInStreams = 0;
  CNum numOutStreams = 0;
  CNum i;
  for (i = 0; i < numCoders; i++)
  {
    folder.Coders.Add(CCoderInfo());
    CCoderInfo &coder = folder.Coders.Back();

    {
      Byte mainByte = ReadByte();
      int idSize = (mainByte & 0xF);
      Byte longID[15];
      ReadBytes(longID, idSize);
      if (idSize > 8)
        ThrowUnsupported();
      UInt64 id = 0;
      for (int j = 0; j < idSize; j++)
        id |= (UInt64)longID[idSize - 1 - j] << (8 * j);
      coder.MethodID = id;

      if ((mainByte & 0x10) != 0)
      {
        coder.NumInStreams = ReadNum();
        coder.NumOutStreams = ReadNum();
      }
      else
      {
        coder.NumInStreams = 1;
        coder.NumOutStreams = 1;
      }
      if ((mainByte & 0x20) != 0)
      {
        CNum propsSize = ReadNum();
        coder.Props.SetCapacity((size_t)propsSize);
        ReadBytes((Byte *)coder.Props, (size_t)propsSize);
      }
      if ((mainByte & 0x80) != 0)
        ThrowUnsupported();
    }
    numInStreams += coder.NumInStreams;
    numOutStreams += coder.NumOutStreams;
  }

  CNum numBindPairs = numOutStreams - 1;
  folder.BindPairs.Clear();
  folder.BindPairs.Reserve(numBindPairs);
  for (i = 0; i < numBindPairs; i++)
  {
    CBindPair bp;
    bp.InIndex = ReadNum();
    bp.OutIndex = ReadNum();
    folder.BindPairs.Add(bp);
  }

  if (numInStreams < numBindPairs)
    ThrowUnsupported();
  CNum numPackStreams = numInStreams - numBindPairs;
  folder.PackStreams.Reserve(numPackStreams);
  if (numPackStreams == 1)
  {
    for (i = 0; i < numInStreams; i++)
      if (folder.FindBindPairForInStream(i) < 0)
      {
        folder.PackStreams.Add(i);
        break;
      }
    if (folder.PackStreams.Size() != 1)
      ThrowUnsupported();
  }
  else
    for (i = 0; i < numPackStreams; i++)
      folder.PackStreams.Add(ReadNum());
}

void CInArchive::WaitAttribute(UInt64 attribute)
{
  for (;;)
  {
    UInt64 type = ReadID();
    if (type == attribute)
      return;
    if (type == NID::kEnd)
      ThrowIncorrect();
    SkipData();
  }
}

void CInArchive::ReadHashDigests(int numItems,
    CBoolVector &digestsDefined,
    CRecordVector<UInt32> &digests)
{
  ReadBoolVector2(numItems, digestsDefined);
  digests.Clear();
  digests.Reserve(numItems);
  for (int i = 0; i < numItems; i++)
  {
    UInt32 crc = 0;
    if (digestsDefined[i])
      crc = ReadUInt32();
    digests.Add(crc);
  }
}

void CInArchive::ReadPackInfo(
    UInt64 &dataOffset,
    CRecordVector<UInt64> &packSizes,
    CBoolVector &packCRCsDefined,
    CRecordVector<UInt32> &packCRCs)
{
  dataOffset = ReadNumber();
  CNum numPackStreams = ReadNum();

  WaitAttribute(NID::kSize);
  packSizes.Clear();
  packSizes.Reserve(numPackStreams);
  for (CNum i = 0; i < numPackStreams; i++)
    packSizes.Add(ReadNumber());

  UInt64 type;
  for (;;)
  {
    type = ReadID();
    if (type == NID::kEnd)
      break;
    if (type == NID::kCRC)
    {
      ReadHashDigests(numPackStreams, packCRCsDefined, packCRCs);
      continue;
    }
    SkipData();
  }
  if (packCRCsDefined.IsEmpty())
  {
    BoolVector_Fill_False(packCRCsDefined, numPackStreams);
    packCRCs.Reserve(numPackStreams);
    packCRCs.Clear();
    for (CNum i = 0; i < numPackStreams; i++)
      packCRCs.Add(0);
  }
}

void CInArchive::ReadUnpackInfo(
    const CObjectVector<CByteBuffer> *dataVector,
    CObjectVector<CFolder> &folders)
{
  WaitAttribute(NID::kFolder);
  CNum numFolders = ReadNum();

  {
    CStreamSwitch streamSwitch;
    streamSwitch.Set(this, dataVector);
    folders.Clear();
    folders.Reserve(numFolders);
    for (CNum i = 0; i < numFolders; i++)
    {
      folders.Add(CFolder());
      GetNextFolderItem(folders.Back());
    }
  }

  WaitAttribute(NID::kCodersUnpackSize);

  CNum i;
  for (i = 0; i < numFolders; i++)
  {
    CFolder &folder = folders[i];
    CNum numOutStreams = folder.GetNumOutStreams();
    folder.UnpackSizes.Reserve(numOutStreams);
    for (CNum j = 0; j < numOutStreams; j++)
      folder.UnpackSizes.Add(ReadNumber());
  }

  for (;;)
  {
    UInt64 type = ReadID();
    if (type == NID::kEnd)
      return;
    if (type == NID::kCRC)
    {
      CBoolVector crcsDefined;
      CRecordVector<UInt32> crcs;
      ReadHashDigests(numFolders, crcsDefined, crcs);
      for (i = 0; i < numFolders; i++)
      {
        CFolder &folder = folders[i];
        folder.UnpackCRCDefined = crcsDefined[i];
        folder.UnpackCRC = crcs[i];
      }
      continue;
    }
    SkipData();
  }
}

void CInArchive::ReadSubStreamsInfo(
    const CObjectVector<CFolder> &folders,
    CRecordVector<CNum> &numUnpackStreamsInFolders,
    CRecordVector<UInt64> &unpackSizes,
    CBoolVector &digestsDefined,
    CRecordVector<UInt32> &digests)
{
  numUnpackStreamsInFolders.Clear();
  numUnpackStreamsInFolders.Reserve(folders.Size());
  UInt64 type;
  for (;;)
  {
    type = ReadID();
    if (type == NID::kNumUnpackStream)
    {
      for (int i = 0; i < folders.Size(); i++)
        numUnpackStreamsInFolders.Add(ReadNum());
      continue;
    }
    if (type == NID::kCRC || type == NID::kSize)
      break;
    if (type == NID::kEnd)
      break;
    SkipData();
  }

  if (numUnpackStreamsInFolders.IsEmpty())
    for (int i = 0; i < folders.Size(); i++)
      numUnpackStreamsInFolders.Add(1);

  int i;
  for (i = 0; i < numUnpackStreamsInFolders.Size(); i++)
  {
    // v3.13 incorrectly worked with empty folders
    // v4.07: we check that folder is empty
    CNum numSubstreams = numUnpackStreamsInFolders[i];
    if (numSubstreams == 0)
      continue;
    UInt64 sum = 0;
    for (CNum j = 1; j < numSubstreams; j++)
      if (type == NID::kSize)
      {
        UInt64 size = ReadNumber();
        unpackSizes.Add(size);
        sum += size;
      }
    unpackSizes.Add(folders[i].GetUnpackSize() - sum);
  }
  if (type == NID::kSize)
    type = ReadID();

  int numDigests = 0;
  int numDigestsTotal = 0;
  for (i = 0; i < folders.Size(); i++)
  {
    CNum numSubstreams = numUnpackStreamsInFolders[i];
    if (numSubstreams != 1 || !folders[i].UnpackCRCDefined)
      numDigests += numSubstreams;
    numDigestsTotal += numSubstreams;
  }

  for (;;)
  {
    if (type == NID::kCRC)
    {
      CBoolVector digestsDefined2;
      CRecordVector<UInt32> digests2;
      ReadHashDigests(numDigests, digestsDefined2, digests2);
      int digestIndex = 0;
      for (i = 0; i < folders.Size(); i++)
      {
        CNum numSubstreams = numUnpackStreamsInFolders[i];
        const CFolder &folder = folders[i];
        if (numSubstreams == 1 && folder.UnpackCRCDefined)
        {
          digestsDefined.Add(true);
          digests.Add(folder.UnpackCRC);
        }
        else
          for (CNum j = 0; j < numSubstreams; j++, digestIndex++)
          {
            digestsDefined.Add(digestsDefined2[digestIndex]);
            digests.Add(digests2[digestIndex]);
          }
      }
    }
    else if (type == NID::kEnd)
    {
      if (digestsDefined.IsEmpty())
      {
        BoolVector_Fill_False(digestsDefined, numDigestsTotal);
        digests.Clear();
        for (int i = 0; i < numDigestsTotal; i++)
          digests.Add(0);
      }
      return;
    }
    else
      SkipData();
    type = ReadID();
  }
}

void CInArchive::ReadStreamsInfo(
    const CObjectVector<CByteBuffer> *dataVector,
    UInt64 &dataOffset,
    CRecordVector<UInt64> &packSizes,
    CBoolVector &packCRCsDefined,
    CRecordVector<UInt32> &packCRCs,
    CObjectVector<CFolder> &folders,
    CRecordVector<CNum> &numUnpackStreamsInFolders,
    CRecordVector<UInt64> &unpackSizes,
    CBoolVector &digestsDefined,
    CRecordVector<UInt32> &digests)
{
  for (;;)
  {
    UInt64 type = ReadID();
    if (type > ((UInt32)1 << 30))
      ThrowIncorrect();
    switch((UInt32)type)
    {
      case NID::kEnd:
        return;
      case NID::kPackInfo:
      {
        ReadPackInfo(dataOffset, packSizes, packCRCsDefined, packCRCs);
        break;
      }
      case NID::kUnpackInfo:
      {
        ReadUnpackInfo(dataVector, folders);
        break;
      }
      case NID::kSubStreamsInfo:
      {
        ReadSubStreamsInfo(folders, numUnpackStreamsInFolders,
            unpackSizes, digestsDefined, digests);
        break;
      }
      default:
        ThrowIncorrect();
    }
  }
}

void CInArchive::ReadBoolVector(int numItems, CBoolVector &v)
{
  v.Clear();
  v.Reserve(numItems);
  Byte b = 0;
  Byte mask = 0;
  for (int i = 0; i < numItems; i++)
  {
    if (mask == 0)
    {
      b = ReadByte();
      mask = 0x80;
    }
    v.Add((b & mask) != 0);
    mask >>= 1;
  }
}

void CInArchive::ReadBoolVector2(int numItems, CBoolVector &v)
{
  Byte allAreDefined = ReadByte();
  if (allAreDefined == 0)
  {
    ReadBoolVector(numItems, v);
    return;
  }
  v.Clear();
  v.Reserve(numItems);
  for (int i = 0; i < numItems; i++)
    v.Add(true);
}

void CInArchive::ReadUInt64DefVector(const CObjectVector<CByteBuffer> &dataVector,
    CUInt64DefVector &v, int numFiles)
{
  ReadBoolVector2(numFiles, v.Defined);

  CStreamSwitch streamSwitch;
  streamSwitch.Set(this, &dataVector);
  v.Values.Reserve(numFiles);

  for (int i = 0; i < numFiles; i++)
  {
    UInt64 t = 0;
    if (v.Defined[i])
      t = ReadUInt64();
    v.Values.Add(t);
  }
}

HRESULT CInArchive::ReadAndDecodePackedStreams(
    DECL_EXTERNAL_CODECS_LOC_VARS
    UInt64 baseOffset,
    UInt64 &dataOffset, CObjectVector<CByteBuffer> &dataVector
    #ifndef _NO_CRYPTO
    , ICryptoGetTextPassword *getTextPassword, bool &passwordIsDefined
    #endif
    )
{
  CRecordVector<UInt64> packSizes;
  CBoolVector packCRCsDefined;
  CRecordVector<UInt32> packCRCs;
  CObjectVector<CFolder> folders;
  
  CRecordVector<CNum> numUnpackStreamsInFolders;
  CRecordVector<UInt64> unpackSizes;
  CBoolVector digestsDefined;
  CRecordVector<UInt32> digests;
  
  ReadStreamsInfo(NULL,
    dataOffset,
    packSizes,
    packCRCsDefined,
    packCRCs,
    folders,
    numUnpackStreamsInFolders,
    unpackSizes,
    digestsDefined,
    digests);
  
  // db.ArchiveInfo.DataStartPosition2 += db.ArchiveInfo.StartPositionAfterHeader;
  
  CNum packIndex = 0;
  CDecoder decoder(
    #ifdef _ST_MODE
    false
    #else
    true
    #endif
    );
  UInt64 dataStartPos = baseOffset + dataOffset;
  for (int i = 0; i < folders.Size(); i++)
  {
    const CFolder &folder = folders[i];
    dataVector.Add(CByteBuffer());
    CByteBuffer &data = dataVector.Back();
    UInt64 unpackSize64 = folder.GetUnpackSize();
    size_t unpackSize = (size_t)unpackSize64;
    if (unpackSize != unpackSize64)
      ThrowUnsupported();
    data.SetCapacity(unpackSize);
    
    CBufPtrSeqOutStream *outStreamSpec = new CBufPtrSeqOutStream;
    CMyComPtr<ISequentialOutStream> outStream = outStreamSpec;
    outStreamSpec->Init(data, unpackSize);
    
    HRESULT result = decoder.Decode(
      EXTERNAL_CODECS_LOC_VARS
      _stream, dataStartPos,
      &packSizes[packIndex], folder, outStream, NULL
      #ifndef _NO_CRYPTO
      , getTextPassword, passwordIsDefined
      #endif
      #if !defined(_7ZIP_ST) && !defined(_SFX)
      , false, 1
      #endif
      );
    RINOK(result);
    
    if (folder.UnpackCRCDefined)
      if (CrcCalc(data, unpackSize) != folder.UnpackCRC)
        ThrowIncorrect();
    for (int j = 0; j < folder.PackStreams.Size(); j++)
    {
      UInt64 packSize = packSizes[packIndex++];
      dataStartPos += packSize;
      HeadersSize += packSize;
    }
  }
  return S_OK;
}

HRESULT CInArchive::ReadHeader(
    DECL_EXTERNAL_CODECS_LOC_VARS
    CArchiveDatabaseEx &db
    #ifndef _NO_CRYPTO
    , ICryptoGetTextPassword *getTextPassword, bool &passwordIsDefined
    #endif
    )
{
  UInt64 type = ReadID();

  if (type == NID::kArchiveProperties)
  {
    ReadArchiveProperties(db.ArchiveInfo);
    type = ReadID();
  }
 
  CObjectVector<CByteBuffer> dataVector;
  
  if (type == NID::kAdditionalStreamsInfo)
  {
    HRESULT result = ReadAndDecodePackedStreams(
        EXTERNAL_CODECS_LOC_VARS
        db.ArchiveInfo.StartPositionAfterHeader,
        db.ArchiveInfo.DataStartPosition2,
        dataVector
        #ifndef _NO_CRYPTO
        , getTextPassword, passwordIsDefined
        #endif
        );
    RINOK(result);
    db.ArchiveInfo.DataStartPosition2 += db.ArchiveInfo.StartPositionAfterHeader;
    type = ReadID();
  }

  CRecordVector<UInt64> unpackSizes;
  CBoolVector digestsDefined;
  CRecordVector<UInt32> digests;
  
  if (type == NID::kMainStreamsInfo)
  {
    ReadStreamsInfo(&dataVector,
        db.ArchiveInfo.DataStartPosition,
        db.PackSizes,
        db.PackCRCsDefined,
        db.PackCRCs,
        db.Folders,
        db.NumUnpackStreamsVector,
        unpackSizes,
        digestsDefined,
        digests);
    db.ArchiveInfo.DataStartPosition += db.ArchiveInfo.StartPositionAfterHeader;
    type = ReadID();
  }
  else
  {
    for (int i = 0; i < db.Folders.Size(); i++)
    {
      db.NumUnpackStreamsVector.Add(1);
      CFolder &folder = db.Folders[i];
      unpackSizes.Add(folder.GetUnpackSize());
      digestsDefined.Add(folder.UnpackCRCDefined);
      digests.Add(folder.UnpackCRC);
    }
  }

  db.Files.Clear();

  if (type == NID::kEnd)
    return S_OK;
  if (type != NID::kFilesInfo)
    ThrowIncorrect();
  
  CNum numFiles = ReadNum();
  db.Files.Reserve(numFiles);
  CNum i;
  for (i = 0; i < numFiles; i++)
    db.Files.Add(CFileItem());

  db.ArchiveInfo.FileInfoPopIDs.Add(NID::kSize);
  if (!db.PackSizes.IsEmpty())
    db.ArchiveInfo.FileInfoPopIDs.Add(NID::kPackInfo);
  if (numFiles > 0  && !digests.IsEmpty())
    db.ArchiveInfo.FileInfoPopIDs.Add(NID::kCRC);

  CBoolVector emptyStreamVector;
  BoolVector_Fill_False(emptyStreamVector, (int)numFiles);
  CBoolVector emptyFileVector;
  CBoolVector antiFileVector;
  CNum numEmptyStreams = 0;

  for (;;)
  {
    UInt64 type = ReadID();
    if (type == NID::kEnd)
      break;
    UInt64 size = ReadNumber();
    size_t ppp = _inByteBack->_pos;
    bool addPropIdToList = true;
    bool isKnownType = true;
    if (type > ((UInt32)1 << 30))
      isKnownType = false;
    else switch((UInt32)type)
    {
      case NID::kName:
      {
        CStreamSwitch streamSwitch;
        streamSwitch.Set(this, &dataVector);
        for (int i = 0; i < db.Files.Size(); i++)
          _inByteBack->ReadString(db.Files[i].Name);
        break;
      }
      case NID::kWinAttributes:
      {
        CBoolVector boolVector;
        ReadBoolVector2(db.Files.Size(), boolVector);
        CStreamSwitch streamSwitch;
        streamSwitch.Set(this, &dataVector);
        for (i = 0; i < numFiles; i++)
        {
          CFileItem &file = db.Files[i];
          file.AttribDefined = boolVector[i];
          if (file.AttribDefined)
            file.Attrib = ReadUInt32();
        }
        break;
      }
      case NID::kEmptyStream:
      {
        ReadBoolVector(numFiles, emptyStreamVector);
        for (i = 0; i < (CNum)emptyStreamVector.Size(); i++)
          if (emptyStreamVector[i])
            numEmptyStreams++;

        BoolVector_Fill_False(emptyFileVector, numEmptyStreams);
        BoolVector_Fill_False(antiFileVector, numEmptyStreams);

        break;
      }
      case NID::kEmptyFile:  ReadBoolVector(numEmptyStreams, emptyFileVector); break;
      case NID::kAnti:  ReadBoolVector(numEmptyStreams, antiFileVector); break;
      case NID::kStartPos:  ReadUInt64DefVector(dataVector, db.StartPos, (int)numFiles); break;
      case NID::kCTime:  ReadUInt64DefVector(dataVector, db.CTime, (int)numFiles); break;
      case NID::kATime:  ReadUInt64DefVector(dataVector, db.ATime, (int)numFiles); break;
      case NID::kMTime:  ReadUInt64DefVector(dataVector, db.MTime, (int)numFiles); break;
      case NID::kDummy:
      {
        for (UInt64 j = 0; j < size; j++)
          if (ReadByte() != 0)
            ThrowIncorrect();
        addPropIdToList = false;
        break;
      }
      default:
        addPropIdToList = isKnownType = false;
    }
    if (isKnownType)
    {
      if(addPropIdToList)
        db.ArchiveInfo.FileInfoPopIDs.Add(type);
    }
    else
      SkipData(size);
    bool checkRecordsSize = (db.ArchiveInfo.Version.Major > 0 ||
        db.ArchiveInfo.Version.Minor > 2);
    if (checkRecordsSize && _inByteBack->_pos - ppp != size)
      ThrowIncorrect();
  }

  CNum emptyFileIndex = 0;
  CNum sizeIndex = 0;

  CNum numAntiItems = 0;
  for (i = 0; i < numEmptyStreams; i++)
    if (antiFileVector[i])
      numAntiItems++;
    
  for (i = 0; i < numFiles; i++)
  {
    CFileItem &file = db.Files[i];
    bool isAnti;
    file.HasStream = !emptyStreamVector[i];
    if (file.HasStream)
    {
      file.IsDir = false;
      isAnti = false;
      file.Size = unpackSizes[sizeIndex];
      file.Crc = digests[sizeIndex];
      file.CrcDefined = digestsDefined[sizeIndex];
      sizeIndex++;
    }
    else
    {
      file.IsDir = !emptyFileVector[emptyFileIndex];
      isAnti = antiFileVector[emptyFileIndex];
      emptyFileIndex++;
      file.Size = 0;
      file.CrcDefined = false;
    }
    if (numAntiItems != 0)
      db.IsAnti.Add(isAnti);
  }
  return S_OK;
}


void CArchiveDatabaseEx::FillFolderStartPackStream()
{
  FolderStartPackStreamIndex.Clear();
  FolderStartPackStreamIndex.Reserve(Folders.Size());
  CNum startPos = 0;
  for (int i = 0; i < Folders.Size(); i++)
  {
    FolderStartPackStreamIndex.Add(startPos);
    startPos += (CNum)Folders[i].PackStreams.Size();
  }
}

void CArchiveDatabaseEx::FillStartPos()
{
  PackStreamStartPositions.Clear();
  PackStreamStartPositions.Reserve(PackSizes.Size());
  UInt64 startPos = 0;
  for (int i = 0; i < PackSizes.Size(); i++)
  {
    PackStreamStartPositions.Add(startPos);
    startPos += PackSizes[i];
  }
}

void CArchiveDatabaseEx::FillFolderStartFileIndex()
{
  FolderStartFileIndex.Clear();
  FolderStartFileIndex.Reserve(Folders.Size());
  FileIndexToFolderIndexMap.Clear();
  FileIndexToFolderIndexMap.Reserve(Files.Size());
  
  int folderIndex = 0;
  CNum indexInFolder = 0;
  for (int i = 0; i < Files.Size(); i++)
  {
    const CFileItem &file = Files[i];
    bool emptyStream = !file.HasStream;
    if (emptyStream && indexInFolder == 0)
    {
      FileIndexToFolderIndexMap.Add(kNumNoIndex);
      continue;
    }
    if (indexInFolder == 0)
    {
      // v3.13 incorrectly worked with empty folders
      // v4.07: Loop for skipping empty folders
      for (;;)
      {
        if (folderIndex >= Folders.Size())
          ThrowIncorrect();
        FolderStartFileIndex.Add(i); // check it
        if (NumUnpackStreamsVector[folderIndex] != 0)
          break;
        folderIndex++;
      }
    }
    FileIndexToFolderIndexMap.Add(folderIndex);
    if (emptyStream)
      continue;
    indexInFolder++;
    if (indexInFolder >= NumUnpackStreamsVector[folderIndex])
    {
      folderIndex++;
      indexInFolder = 0;
    }
  }
}

HRESULT CInArchive::ReadDatabase2(
    DECL_EXTERNAL_CODECS_LOC_VARS
    CArchiveDatabaseEx &db
    #ifndef _NO_CRYPTO
    , ICryptoGetTextPassword *getTextPassword, bool &passwordIsDefined
    #endif
    )
{
  db.Clear();
  db.ArchiveInfo.StartPosition = _arhiveBeginStreamPosition;

  db.ArchiveInfo.Version.Major = _header[6];
  db.ArchiveInfo.Version.Minor = _header[7];

  if (db.ArchiveInfo.Version.Major != kMajorVersion)
    ThrowUnsupportedVersion();

  UInt32 crcFromArchive = Get32(_header + 8);
  UInt64 nextHeaderOffset = Get64(_header + 0xC);
  UInt64 nextHeaderSize = Get64(_header + 0x14);
  UInt32 nextHeaderCRC = Get32(_header + 0x1C);
  UInt32 crc = CrcCalc(_header + 0xC, 20);

  #ifdef FORMAT_7Z_RECOVERY
  if (crcFromArchive == 0 && nextHeaderOffset == 0 && nextHeaderSize == 0 && nextHeaderCRC == 0)
  {
    UInt64 cur, cur2;
    RINOK(_stream->Seek(0, STREAM_SEEK_CUR, &cur));
    const int kCheckSize = 500;
    Byte buf[kCheckSize];
    RINOK(_stream->Seek(0, STREAM_SEEK_END, &cur2));
    int checkSize = kCheckSize;
    if (cur2 - cur < kCheckSize)
      checkSize = (int)(cur2 - cur);
    RINOK(_stream->Seek(-checkSize, STREAM_SEEK_END, &cur2));
    
    RINOK(ReadStream_FALSE(_stream, buf, (size_t)checkSize));

    int i;
    for (i = (int)checkSize - 2; i >= 0; i--)
      if (buf[i] == 0x17 && buf[i + 1] == 0x6 || buf[i] == 0x01 && buf[i + 1] == 0x04)
        break;
    if (i < 0)
      return S_FALSE;
    nextHeaderSize = checkSize - i;
    nextHeaderOffset = cur2 - cur + i;
    nextHeaderCRC = CrcCalc(buf + i, (size_t)nextHeaderSize);
    RINOK(_stream->Seek(cur, STREAM_SEEK_SET, NULL));
  }
  else
  #endif
  {
    if (crc != crcFromArchive)
      ThrowIncorrect();
  }

  db.ArchiveInfo.StartPositionAfterHeader = _arhiveBeginStreamPosition + kHeaderSize;

  if (nextHeaderSize == 0)
    return S_OK;

  if (nextHeaderSize > (UInt64)0xFFFFFFFF)
    return S_FALSE;

  if ((Int64)nextHeaderOffset < 0)
    return S_FALSE;

  RINOK(_stream->Seek(nextHeaderOffset, STREAM_SEEK_CUR, NULL));

  CByteBuffer buffer2;
  buffer2.SetCapacity((size_t)nextHeaderSize);

  RINOK(ReadStream_FALSE(_stream, buffer2, (size_t)nextHeaderSize));
  HeadersSize += kHeaderSize + nextHeaderSize;
  db.PhySize = kHeaderSize + nextHeaderOffset + nextHeaderSize;

  if (CrcCalc(buffer2, (UInt32)nextHeaderSize) != nextHeaderCRC)
    ThrowIncorrect();
  
  CStreamSwitch streamSwitch;
  streamSwitch.Set(this, buffer2);
  
  CObjectVector<CByteBuffer> dataVector;
  
  UInt64 type = ReadID();
  if (type != NID::kHeader)
  {
    if (type != NID::kEncodedHeader)
      ThrowIncorrect();
    HRESULT result = ReadAndDecodePackedStreams(
        EXTERNAL_CODECS_LOC_VARS
        db.ArchiveInfo.StartPositionAfterHeader,
        db.ArchiveInfo.DataStartPosition2,
        dataVector
        #ifndef _NO_CRYPTO
        , getTextPassword, passwordIsDefined
        #endif
        );
    RINOK(result);
    if (dataVector.Size() == 0)
      return S_OK;
    if (dataVector.Size() > 1)
      ThrowIncorrect();
    streamSwitch.Remove();
    streamSwitch.Set(this, dataVector.Front());
    if (ReadID() != NID::kHeader)
      ThrowIncorrect();
  }

  db.HeadersSize = HeadersSize;

  return ReadHeader(
    EXTERNAL_CODECS_LOC_VARS
    db
    #ifndef _NO_CRYPTO
    , getTextPassword, passwordIsDefined
    #endif
    );
}

HRESULT CInArchive::ReadDatabase(
    DECL_EXTERNAL_CODECS_LOC_VARS
    CArchiveDatabaseEx &db
    #ifndef _NO_CRYPTO
    , ICryptoGetTextPassword *getTextPassword, bool &passwordIsDefined
    #endif
    )
{
  try
  {
    return ReadDatabase2(
      EXTERNAL_CODECS_LOC_VARS db
      #ifndef _NO_CRYPTO
      , getTextPassword, passwordIsDefined
      #endif
      );
  }
  catch(CInArchiveException &) { return S_FALSE; }
}

}}