manifest

package module
v1.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 24, 2024 License: MIT Imports: 13 Imported by: 1

README

Epic Games Manifest Parser

Epic games is using manifest files to download or update games

Installation

go get github.com/meszmate/manifest

Example

If you want to use this libary, you need to understand how epic games is downloading and updating games. You can check Epic Games alternative libaries on github, but here's a fast example of installing the files.

package main

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"time"

	manifest "github.com/meszmate/manifest"
	chunks "github.com/meszmate/manifest/chunks"
)

const downloadpath string = "/Users/meszmate/manifestparse/" // Leave it empty if you want to install in the current directory
const max_retries int = 7

func main(){
	stime := time.Now().Unix()
	filebytes := manifest.LoadFileBytes("/Users/meszmate/Downloads/889Cfv4W7UAZ6Jn0dUyIuV0kX7gTog.manifest") // manifest.LoadURLBytes("url") if you want to get the manifest from url
	manifestreader := bytes.NewReader(filebytes)
	binary, err := manifest.ParseManifest(manifestreader)
	if err != nil{
		fmt.Println(err)
	}
	for _, i := range binary.FileManifestList.FileManifestList{
		if !manifest.StringContains3(i.InstallTags, []string{"br_highres", "stw_highres", "core_highres", "ondemand", "sm6"}){ 
			fpath := downloadpath + i.FileName
			err := os.MkdirAll(filepath.Dir(fpath), os.ModePerm)
			if err != nil {
				log.Fatalf("Failed to create directories: %v", err)
			}
			file, err := os.Create(fpath)
			if err != nil {
				log.Fatalf("Failed to create file: %v", err)
			}
			defer file.Close()
			for _, x := range i.ChunkParts{
				newbytes := getChunkByURL(x.Chunk.GetURL("http://epicgames-download1.akamaized.net/Builds/Fortnite/CloudDir/" + binary.Metadata.FeatureLevel.ChunkSubDir()), max_retries) 
				if newbytes != nil{
					newdata, err := chunks.Decompress(newbytes)
					if err != nil{
						log.Fatal("Failed to decompress: " + err.Error())
					}
					file.Write(newdata[x.Offset:x.Offset+x.Size])
				}else{
					fmt.Println("Failed to get a chunk, next...")
				}
			}
			fmt.Println(i.FileName + " Successfully installed")
		}
	}
	etime := time.Now().Unix()
	fmt.Printf("Installed in %d seconds\n", etime-stime)
}

func getChunkByURL(url string, retries int) []byte{
	retry := 0
	for retry < retries+1{
		newbytes := manifest.LoadURLBytes(url)
		if newbytes != nil{
			return newbytes
		}
		retry++
	}
	return nil
}

Chunk Download BaseURLs Performance

http://download.epicgames.com/Builds/Fortnite/CloudDir/       		20-21 ms
http://cloudflare.epicgamescdn.com/Builds/Fortnite/CloudDir/  		34-36 ms
http://fastly-download.epicgames.com/Builds/Fortnite/CloudDir/ 		19-20 ms
http://epicgames-download1.akamaized.net/Builds/Fortnite/CloudDir/      27-28 ms

Manifest ApplyDelta Usage

When you get the manifest from epic games api, you will get "elements" for manifest, you have to choose one "uri", and that will be the delta manifest baseURL. Example:

newManifest, _ := manifest.ParseManifest(...)
oldManifest, _ := manifest.ParseManifest(...)
deltaManifestBytes := manifest.GetDeltaManifest("the base url", new_manifest.Metadata.BuildId, old_manifest.Metadata.BuildId)
deltaManifestReader := bytes.NewReader(deltaManifestBytes)
deltaManifest, _ := manifest.ParseManifest(deltaManifestReader)
newManifest.ApplyDelta(deltaManifest)

How updating is working?

for updating a game, you will need the old manifest + you can use applyDelta function for optimizing the NEW manifest. Changed chunks will have different guid.

Documentation

Index

Constants

View Source 
const (
	StoredCompressed uint8 = 0x01
	StoredEncrypted  uint8 = 0x02
)
View Source 
const BinaryManifestMagic = 0x44BEC00C

Variables

View Source 
var (
	ErrBadMagic = errors.New("bad magic found, must be 0x44BEC00C")
)

Functions

func GetDeltaManifest 

func GetDeltaManifest(baseURL string, newBuildID string, oldBuildID string) []byte

func LoadFileBytes 

func LoadFileBytes(path string) []byte

func LoadURLBytes 

func LoadURLBytes(url string) []byte

func StringContains 

func StringContains(array []string, s string) bool

func StringContains2 

func StringContains2(array []string, array2 []string) bool

func StringContains3 

func StringContains3(array []string, array2 []string) bool

Types

type BinaryManifest 

type BinaryManifest struct {
	Header           *FManifestHeader
	Metadata         *FManifestMeta
	ChunkDataList    *FChunkDataList
	FileManifestList *FFileManifestList
	CustomFields     *FCustomFields
}

func ParseManifest 

func ParseManifest(f io.ReadSeeker) (*BinaryManifest, error)

func (*BinaryManifest) ApplyDelta 

func (m *BinaryManifest) ApplyDelta(deltaManifest *BinaryManifest)

type Chunk 

type Chunk struct {
	GUID       uuid.UUID
	Hash       uint64
	SHAHash    [20]byte
	Group      uint8
	WindowSize uint32
	FileSize   uint64
}

func (*Chunk) GetURL 

func (c *Chunk) GetURL(chunksDir string) string

gets the URL for a chunk. example for chunksDir: http://epicgames-download1.akamaized.net/Builds/Fortnite/CloudDir/ChunksV4

type ChunkPart 

type ChunkPart struct {
	DataSize   uint32
	ParentGUID uuid.UUID
	Offset     uint32
	Size       uint32

	Chunk *Chunk
}

type EFeatureLevel 

type EFeatureLevel int32
const (
	// The original version.
	EFeatureLevelOriginal EFeatureLevel = iota + 0
	// Support for custom fields.
	EFeatureLevelCustomFields
	// Started storing the version number.
	EFeatureLevelStartStoringVersion
	// Made after data files where renamed to include the hash value, these chunks now go to ChunksV2.
	EFeatureLevelDataFileRenames
	// Manifest stores whether build was constructed with chunk or file data.
	EFeatureLevelStoresIfChunkOrFileData
	// Manifest stores group number for each chunk/file data for reference so that external readers don't need to know how to calculate them.
	EFeatureLevelStoresDataGroupNumbers
	// Added support for chunk compression, these chunks now go to ChunksV3. NB: Not File Data Compression yet.
	EFeatureLevelChunkCompressionSupport
	// Manifest stores product prerequisites info.
	EFeatureLevelStoresPrerequisitesInfo
	// Manifest stores chunk download sizes.
	EFeatureLevelStoresChunkFileSizes
	// Manifest can optionally be stored using UObject serialization and compressed.
	EFeatureLevelStoredAsCompressedUClass
	// These two features were removed and never used.
	EFeatureLevelUNUSED_0
	EFeatureLevelUNUSED_1
	// Manifest stores chunk data SHA1 hash to use in place of data compare, for faster generation.
	EFeatureLevelStoresChunkDataShaHashes
	// Manifest stores Prerequisite Ids.
	EFeatureLevelStoresPrerequisiteIds
	// The first minimal binary format was added. UObject classes will no longer be saved out when binary selected.
	EFeatureLevelStoredAsBinaryData
	// Temporary level where manifest can reference chunks with dynamic window size, but did not serialize them. Chunks from here onwards are stored in ChunksV4.
	EFeatureLevelVariableSizeChunksWithoutWindowSizeChunkInfo
	// Manifest can reference chunks with dynamic window size, and also serializes them.
	EFeatureLevelVariableSizeChunks
	// Manifest stores a unique build id for exact matching of build data.
	EFeatureLevelStoresUniqueBuildId
	// !! Always after the latest version entry, signifies the latest version plus 1 to allow the following Latest alias.
	EFeatureLevelLatestPlusOne
	// An alias for the actual latest version value.
	EFeatureLevelLatest = (EFeatureLevelLatestPlusOne - 1)
	// An alias to provide the latest version of a manifest supported by file data (nochunks).
	LatestNoChunks = EFeatureLevelStoresChunkFileSizes
	// An alias to provide the latest version of a manifest supported by a json serialized format.
	LatestJson = EFeatureLevelStoresPrerequisiteIds
	// An alias to provide the first available version of optimised delta manifest saving.
	FirstOptimisedDelta = EFeatureLevelStoresUniqueBuildId
	// JSON manifests were stored with a version of 255 during a certain CL range due to a bug.
	// We will treat this as being StoresChunkFileSizes in code.
	EFeatureLevelBrokenJsonVersion = 255
	// This is for UObject default, so that we always serialize it.
	EFeatureLevelInvalid = -1
)

func (EFeatureLevel) ChunkSubDir 

func (e EFeatureLevel) ChunkSubDir() string

ChunkSubDir returns the chunk version sub directory

source: https://github.com/EpicGames/UnrealEngine/blob/d9d435c9c280b99a6c679b517adedd3f4b02cfd7/Engine/Source/Runtime/Online/BuildPatchServices/Private/Data/ManifestData.cpp#L77

func (EFeatureLevel) String 

func (i EFeatureLevel) String() string

type FChunkDataList 

type FChunkDataList struct {
	DataSize    uint32
	DataVersion uint8
	Count       uint32

	Chunks      []*Chunk
	ChunkLookup map[uuid.UUID]uint32
}

func ReadChunkDataList 

func ReadChunkDataList(f io.ReadSeeker) (*FChunkDataList, error)

type FCustomFields 

type FCustomFields struct {
	DataSize    uint32
	DataVersion uint8
	Count       uint32
	Fields      map[string]string
}

func ReadCustomFields 

func ReadCustomFields(f io.ReadSeeker) (*FCustomFields, error)

type FFileManifestList 

type FFileManifestList struct {
	DataSize    uint32
	DataVersion uint8
	Count       uint32

	FileManifestList []File
}

func ReadFileManifestList 

func ReadFileManifestList(f io.ReadSeeker, dataList *FChunkDataList) (*FFileManifestList, error)

func (*FFileManifestList) GetFileByPath 

func (f *FFileManifestList) GetFileByPath(p string) *File

type FManifestHeader 

type FManifestHeader struct {
	HeaderSize           int32
	DataSizeUncompressed int32
	DataSizeCompressed   int32
	SHAHash              [20]byte
	StoredAs             uint8
	Version              EFeatureLevel
}

func ParseHeader 

func ParseHeader(f io.ReadSeeker) (*FManifestHeader, error)

func (FManifestHeader) String 

func (h FManifestHeader) String() string

type FManifestMeta 

type FManifestMeta struct {
	DataSize    uint32
	DataVersion uint8

	FeatureLevel  EFeatureLevel
	IsFileData    bool
	AppID         int32
	AppName       string
	BuildVersion  string
	LaunchExe     string
	LaunchCommand string
	PrereqIds     []string
	PrereqName    string
	PrereqPath    string
	PrereqArgs    string

	// if DataVersion >= 1
	BuildId string
}

func ReadFManifestMeta 

func ReadFManifestMeta(f io.ReadSeeker) (*FManifestMeta, error)

func (FManifestMeta) String 

func (m FManifestMeta) String() string

type File 

type File struct {
	FileName      string
	SymlinkTarget string
	SHAHash       [20]byte
	FileMetaFlags uint8
	InstallTags   []string
	FileSize      uint32

	ChunkParts []ChunkPart
}

TODO: implement io.ReadSeeker on this

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.