Documentation
¶
Index ¶
Constants ¶
This section is empty.
Variables ¶
View Source
var BacktestCmd = &cobra.Command{ Use: "backtest", Short: "run backtest with strategies", SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { verboseCnt, err := cmd.Flags().GetCount("verbose") if err != nil { return err } if viper.GetBool("debug") { verboseCnt = 2 } configFile, err := cmd.Flags().GetString("config") if err != nil { return err } if len(configFile) == 0 { return errors.New("--config option is required") } wantBaseAssetBaseline, err := cmd.Flags().GetBool("base-asset-baseline") if err != nil { return err } wantSync, err := cmd.Flags().GetBool("sync") if err != nil { return err } syncExchangeName, err := cmd.Flags().GetString("sync-exchange") if err != nil { return err } sessionName, err := cmd.Flags().GetString("session") if err != nil { return err } force, err := cmd.Flags().GetBool("force") if err != nil { return err } outputDirectory, err := cmd.Flags().GetString("output") if err != nil { return err } generatingReport := len(outputDirectory) > 0 reportFileInSubDir, err := cmd.Flags().GetBool("subdir") if err != nil { return err } syncOnly, err := cmd.Flags().GetBool("sync-only") if err != nil { return err } syncFromDateStr, err := cmd.Flags().GetString("sync-from") if err != nil { return err } shouldVerify, err := cmd.Flags().GetBool("verify") if err != nil { return err } userConfig, err := bbgo.Load(configFile, true) if err != nil { return err } if userConfig.Backtest == nil { return errors.New("backtest config is not defined") } ctx, cancel := context.WithCancel(context.Background()) defer cancel() var now = time.Now() var startTime, endTime time.Time startTime = userConfig.Backtest.StartTime.Time() if userConfig.Backtest.EndTime != nil { endTime = userConfig.Backtest.EndTime.Time() } else { endTime = now } log.Infof("starting backtest with startTime %s", startTime.Format(time.ANSIC)) environ := bbgo.NewEnvironment() if err := BootstrapBacktestEnvironment(ctx, environ, userConfig); err != nil { return err } if environ.DatabaseService == nil { return errors.New("database service is not enabled, please check your environment variables DB_DRIVER and DB_DSN") } backtestService := &service.BacktestService{DB: environ.DatabaseService.DB} environ.BacktestService = backtestService if len(sessionName) > 0 { userConfig.Backtest.Sessions = []string{sessionName} } else if len(syncExchangeName) > 0 { userConfig.Backtest.Sessions = []string{syncExchangeName} } else if len(userConfig.Backtest.Sessions) == 0 { log.Infof("backtest.sessions is not defined, using all supported exchanges: %v", types.SupportedExchanges) for _, exName := range types.SupportedExchanges { userConfig.Backtest.Sessions = append(userConfig.Backtest.Sessions, exName.String()) } } var sourceExchanges = make(map[types.ExchangeName]types.Exchange) for _, name := range userConfig.Backtest.Sessions { exName, err := types.ValidExchangeName(name) if err != nil { return err } publicExchange, err := exchange.NewPublic(exName) if err != nil { return err } sourceExchanges[exName] = publicExchange } var syncFromTime time.Time if len(syncFromDateStr) > 0 { syncFromTime, err = time.Parse(types.DateFormat, syncFromDateStr) if err != nil { return err } if syncFromTime.After(startTime) { return fmt.Errorf("sync-from time %s can not be latter than the backtest start time %s", syncFromTime, startTime) } } else { syncFromTime = startTime.AddDate(0, -1, 0) log.Infof("adjusted sync start time %s to %s for backward market data", startTime, syncFromTime) } if wantSync { log.Infof("starting synchronization: %v", userConfig.Backtest.Symbols) if err := sync(ctx, userConfig, backtestService, sourceExchanges, syncFromTime.Local(), endTime.Local()); err != nil { return err } log.Info("synchronization done") if shouldVerify { err := verify(userConfig, backtestService, sourceExchanges, syncFromTime.Local(), endTime.Local()) if err != nil { return err } } if syncOnly { return nil } } if userConfig.Backtest.RecordTrades { log.Warn("!!! Trade recording is enabled for back-testing !!!") log.Warn("!!! To run back-testing, you should use an isolated database for storing back-testing trades !!!") log.Warn("!!! The trade record in the current database WILL ALL BE DELETED BEFORE THIS BACK-TESTING !!!") if !force { if !confirmation("Are you sure to continue?") { return nil } } if err := environ.TradeService.DeleteAll(); err != nil { return err } } if verboseCnt == 2 { log.SetLevel(log.DebugLevel) } else if verboseCnt > 0 { log.SetLevel(log.InfoLevel) } else { log.SetLevel(log.ErrorLevel) } environ.SetStartTime(startTime) for name, sourceExchange := range sourceExchanges { backtestExchange, err := backtest.NewExchange(sourceExchange.Name(), sourceExchange, backtestService, userConfig.Backtest) if err != nil { return errors.Wrap(err, "failed to create backtest exchange") } session := environ.AddExchange(name.String(), backtestExchange) exchangeFromConfig := userConfig.Sessions[name.String()] if exchangeFromConfig != nil { session.UseHeikinAshi = exchangeFromConfig.UseHeikinAshi } } if err := environ.Init(ctx); err != nil { return err } trader := bbgo.NewTrader(environ) if verboseCnt == 0 { trader.DisableLogging() } if err := trader.Configure(userConfig); err != nil { return err } if err := trader.Run(ctx); err != nil { return err } backTestIntervals := []types.Interval{types.Interval1h, types.Interval1d} exchangeSources, err := toExchangeSources(environ.Sessions(), backTestIntervals...) if err != nil { return err } var kLineHandlers []func(k types.KLine, exSource *backtest.ExchangeDataSource) var manifests backtest.Manifests var runID = userConfig.GetSignature() + "_" + uuid.NewString() var reportDir = outputDirectory if generatingReport { if reportFileInSubDir { reportDir = filepath.Join(reportDir, runID) } if err := util.SafeMkdirAll(reportDir); err != nil { return err } startTimeStr := startTime.Format("20060102") endTimeStr := endTime.Format("20060102") kLineSubDir := strings.Join([]string{"klines", "_", startTimeStr, "-", endTimeStr}, "") kLineDataDir := filepath.Join(outputDirectory, "shared", kLineSubDir) if err := util.SafeMkdirAll(kLineDataDir); err != nil { return err } stateRecorder := backtest.NewStateRecorder(reportDir) err = trader.IterateStrategies(func(st bbgo.StrategyID) error { return stateRecorder.Scan(st.(backtest.Instance)) }) if err != nil { return err } manifests = stateRecorder.Manifests() manifests, err = rewriteManifestPaths(manifests, reportDir) if err != nil { return err } kLineHandlers = append(kLineHandlers, func(k types.KLine, _ *backtest.ExchangeDataSource) { if k.Interval == types.Interval1m && k.Closed { if _, err := stateRecorder.Snapshot(); err != nil { log.WithError(err).Errorf("state record failed to snapshot the strategy state") } } }) dumper := backtest.NewKLineDumper(kLineDataDir) defer func() { _ = dumper.Close() }() defer func() { if err := dumper.Close(); err != nil { log.WithError(err).Errorf("kline dumper can not close files") } }() kLineHandlers = append(kLineHandlers, func(k types.KLine, _ *backtest.ExchangeDataSource) { if err := dumper.Record(k); err != nil { log.WithError(err).Errorf("can not write kline to file") } }) equityCurveTsv, err := tsv.NewWriterFile(filepath.Join(reportDir, "equity_curve.tsv")) if err != nil { return err } defer func() { _ = equityCurveTsv.Close() }() _ = equityCurveTsv.Write([]string{ "time", "in_usd", }) defer equityCurveTsv.Flush() kLineHandlers = append(kLineHandlers, func(k types.KLine, exSource *backtest.ExchangeDataSource) { if k.Interval != types.Interval1h { return } balances, err := exSource.Exchange.QueryAccountBalances(ctx) if err != nil { log.WithError(err).Errorf("query back-test account balance error") } else { assets := balances.Assets(exSource.Session.AllLastPrices(), k.EndTime.Time()) _ = equityCurveTsv.Write([]string{ k.EndTime.Time().Format(time.RFC1123), assets.InUSD().String(), }) } }) ordersTsv, err := tsv.NewWriterFile(filepath.Join(reportDir, "orders.tsv")) if err != nil { return err } defer func() { _ = ordersTsv.Close() }() _ = ordersTsv.Write(types.Order{}.CsvHeader()) for _, exSource := range exchangeSources { exSource.Session.UserDataStream.OnOrderUpdate(func(order types.Order) { if order.Status == types.OrderStatusFilled { for _, record := range order.CsvRecords() { _ = ordersTsv.Write(record) } } }) } } runCtx, cancelRun := context.WithCancel(ctx) go func() { defer cancelRun() // Optimize back-test speed for single exchange source var numOfExchangeSources = len(exchangeSources) if numOfExchangeSources == 1 { exSource := exchangeSources[0] for k := range exSource.C { exSource.Exchange.ConsumeKLine(k) for _, h := range kLineHandlers { h(k, &exSource) } } if err := exSource.Exchange.CloseMarketData(); err != nil { log.WithError(err).Errorf("close market data error") } return } RunMultiExchangeData: for { for _, exK := range exchangeSources { k, more := <-exK.C if !more { if err := exK.Exchange.CloseMarketData(); err != nil { log.WithError(err).Errorf("close market data error") return } break RunMultiExchangeData } exK.Exchange.ConsumeKLine(k) for _, h := range kLineHandlers { h(k, &exK) } } } }() cmdutil.WaitForSignal(runCtx, syscall.SIGINT, syscall.SIGTERM) log.Infof("shutting down trader...") shutdownCtx, cancelShutdown := context.WithDeadline(runCtx, time.Now().Add(10*time.Second)) trader.Graceful.Shutdown(shutdownCtx) cancelShutdown() log.SetLevel(log.InfoLevel) initTotalBalances := types.BalanceMap{} finalTotalBalances := types.BalanceMap{} var sessionNames []string for _, session := range environ.Sessions() { sessionNames = append(sessionNames, session.Name) accountConfig := userConfig.Backtest.GetAccount(session.Name) initBalances := accountConfig.Balances.BalanceMap() initTotalBalances = initTotalBalances.Add(initBalances) finalBalances := session.GetAccount().Balances() finalTotalBalances = finalTotalBalances.Add(finalBalances) } summaryReport := &backtest.SummaryReport{ StartTime: startTime, EndTime: endTime, Sessions: sessionNames, InitialTotalBalances: initTotalBalances, FinalTotalBalances: finalTotalBalances, Manifests: manifests, Symbols: nil, } allKLineIntervals := map[types.Interval]struct{}{} for _, interval := range backTestIntervals { allKLineIntervals[interval] = struct{}{} } for _, session := range environ.Sessions() { for _, sub := range session.Subscriptions { if sub.Channel == types.KLineChannel { allKLineIntervals[sub.Options.Interval] = struct{}{} } } } for interval := range allKLineIntervals { summaryReport.Intervals = append(summaryReport.Intervals, interval) } for _, session := range environ.Sessions() { for symbol, trades := range session.Trades { symbolReport, err := createSymbolReport(userConfig, session, symbol, trades.Trades) if err != nil { return err } summaryReport.Symbols = append(summaryReport.Symbols, symbol) summaryReport.SymbolReports = append(summaryReport.SymbolReports, *symbolReport) summaryReport.TotalProfit = symbolReport.PnL.Profit summaryReport.TotalUnrealizedProfit = symbolReport.PnL.UnrealizedProfit if generatingReport { reportFileName := fmt.Sprintf("symbol_report_%s.json", symbol) if err := util.WriteJsonFile(filepath.Join(reportDir, reportFileName), &symbolReport); err != nil { return err } } } } if generatingReport { summaryReportFile := filepath.Join(reportDir, "summary.json") fmt.Println(summaryReportFile) if err := util.WriteJsonFile(summaryReportFile, summaryReport); err != nil { return err } if reportFileInSubDir { if err := backtest.AddReportIndexRun(outputDirectory, backtest.Run{ ID: runID, Config: userConfig, Time: time.Now(), }); err != nil { return err } } } else { color.Green("BACK-TEST REPORT") color.Green("===============================================\n") color.Green("START TIME: %s\n", startTime.Format(time.RFC1123)) color.Green("END TIME: %s\n", endTime.Format(time.RFC1123)) color.Green("INITIAL TOTAL BALANCE: %v\n", initTotalBalances) color.Green("FINAL TOTAL BALANCE: %v\n", finalTotalBalances) for _, symbolReport := range summaryReport.SymbolReports { symbolReport.Print(wantBaseAssetBaseline) } } return nil }, }
View Source
var BuildCmd = &cobra.Command{ Use: "build", Short: "build cross-platform binary", SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() configFile, err := cmd.Flags().GetString("config") if err != nil { return err } if len(configFile) == 0 { return errors.New("--config option is required") } userConfig, err := bbgo.LoadBuildConfig(configFile) if err != nil { return err } if userConfig.Build == nil { return errors.New("build config is not defined") } for _, target := range userConfig.Build.Targets { log.Infof("building %s ...", target.Name) binary, err := bbgo.BuildTarget(ctx, userConfig, target) if err != nil { return err } log.Infof("build succeeded: %s", binary) } return nil }, }
View Source
var PnLCmd = &cobra.Command{ Use: "pnl", Short: "Average Cost Based PnL Calculator", Long: "This command calculates the average cost-based profit from your total trades", SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() sessionName, err := cmd.Flags().GetString("session") if err != nil { return err } symbol, err := cmd.Flags().GetString("symbol") if err != nil { return err } if len(symbol) == 0 { return errors.New("--symbol [SYMBOL] is required") } limit, err := cmd.Flags().GetInt("limit") if err != nil { return err } environ := bbgo.NewEnvironment() if err := environ.ConfigureDatabase(ctx); err != nil { return err } if err := environ.ConfigureExchangeSessions(userConfig); err != nil { return err } session, ok := environ.Session(sessionName) if !ok { return fmt.Errorf("session %s not found", sessionName) } if err := environ.SyncSession(ctx, session); err != nil { return err } if err = environ.Init(ctx); err != nil { return err } exchange := session.Exchange market, ok := session.Market(symbol) if !ok { return fmt.Errorf("market config %s not found", symbol) } since := time.Now().AddDate(-1, 0, 0) until := time.Now() includeTransfer, err := cmd.Flags().GetBool("include-transfer") if err != nil { return err } if includeTransfer { transferService, ok := exchange.(types.ExchangeTransferService) if !ok { return fmt.Errorf("session exchange %s does not implement transfer service", sessionName) } deposits, err := transferService.QueryDepositHistory(ctx, market.BaseCurrency, since, until) if err != nil { return err } _ = deposits withdrawals, err := transferService.QueryWithdrawHistory(ctx, market.BaseCurrency, since, until) if err != nil { return err } sort.Slice(withdrawals, func(i, j int) bool { a := withdrawals[i].ApplyTime.Time() b := withdrawals[j].ApplyTime.Time() return a.Before(b) }) backtestService := &service.BacktestService{DB: environ.DatabaseService.DB} if err := backtestService.Sync(ctx, exchange, symbol, types.Interval1d, since, until); err != nil { return err } } var trades []types.Trade tradingFeeCurrency := exchange.PlatformFeeCurrency() if strings.HasPrefix(symbol, tradingFeeCurrency) { log.Infof("loading all trading fee currency related trades: %s", symbol) trades, err = environ.TradeService.QueryForTradingFeeCurrency(exchange.Name(), symbol, tradingFeeCurrency) } else { trades, err = environ.TradeService.Query(service.QueryTradesOptions{ Symbol: symbol, Limit: limit, }) } if err != nil { return err } if len(trades) == 0 { return errors.New("empty trades, you need to run sync command to sync the trades from the exchange first") } trades = types.SortTradesAscending(trades) log.Infof("%d trades loaded", len(trades)) tickers, err := exchange.QueryTickers(ctx, symbol) if err != nil { return err } currentTick, ok := tickers[symbol] if !ok { return errors.New("no ticker data for current price") } currentPrice := currentTick.Last calculator := &pnl.AverageCostCalculator{ TradingFeeCurrency: tradingFeeCurrency, } report := calculator.Calculate(symbol, trades, currentPrice) report.Print() log.Warnf("note that if you're using cross-exchange arbitrage, the PnL won't be accurate") log.Warnf("withdrawal and deposits are not considered in the PnL") return nil }, }
View Source
var RootCmd = &cobra.Command{ Use: "bbgo", Short: "bbgo is a crypto trading bot", SilenceUsage: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { if err := cobraLoadDotenv(cmd, args); err != nil { return err } if viper.GetBool("debug") { log.Infof("debug mode is enabled") log.SetLevel(log.DebugLevel) } if viper.GetBool("metrics") { http.Handle("/metrics", promhttp.Handler()) go func() { port := viper.GetString("metrics-port") log.Infof("starting metrics server at :%s", port) err := http.ListenAndServe(":"+port, nil) if err != nil { log.WithError(err).Errorf("metrics server error") } }() } cpuProfile, err := cmd.Flags().GetString("cpu-profile") if err != nil { return err } if cpuProfile != "" { log.Infof("starting cpu profiler...") cpuProfileFile, err = os.Create(cpuProfile) if err != nil { log.Fatal("could not create CPU profile: ", err) } if err := pprof.StartCPUProfile(cpuProfileFile); err != nil { log.Fatal("could not start CPU profile: ", err) } } return cobraLoadConfig(cmd, args) }, PersistentPostRunE: func(cmd *cobra.Command, args []string) error { pprof.StopCPUProfile() if cpuProfileFile != nil { return cpuProfileFile.Close() } return nil }, RunE: func(cmd *cobra.Command, args []string) error { return nil }, }
View Source
var RunCmd = &cobra.Command{ Use: "run", Short: "run strategies from config file", SilenceUsage: true, RunE: run, }
View Source
var SyncCmd = &cobra.Command{ Use: "sync [--session=[exchange_name]] [--symbol=[pair_name]] [[--since=yyyy/mm/dd]]", Short: "sync trades and orders history", SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() configFile, err := cmd.Flags().GetString("config") if err != nil { return err } if len(configFile) == 0 { return errors.New("--config option is required") } if _, err := os.Stat(configFile); os.IsNotExist(err) { return err } userConfig, err := bbgo.Load(configFile, false) if err != nil { return err } since, err := cmd.Flags().GetString("since") if err != nil { return err } environ := bbgo.NewEnvironment() if err := environ.ConfigureDatabase(ctx); err != nil { return err } if err := environ.ConfigureExchangeSessions(userConfig); err != nil { return err } sessionNames, err := cmd.Flags().GetStringArray("session") if err != nil { return err } symbol, err := cmd.Flags().GetString("symbol") if err != nil { return err } var ( // default sync start time defaultSyncStartTime = time.Now().AddDate(-1, 0, 0) ) var syncStartTime = defaultSyncStartTime if userConfig.Sync != nil && userConfig.Sync.Since != nil { syncStartTime = userConfig.Sync.Since.Time() } if len(since) > 0 { syncStartTime, err = time.ParseInLocation("2006-01-02", since, time.Local) if err != nil { return err } } environ.SetSyncStartTime(syncStartTime) if len(symbol) > 0 { if userConfig.Sync != nil && len(userConfig.Sync.Symbols) > 0 { userConfig.Sync.Symbols = []bbgo.SyncSymbol{ {Symbol: symbol}, } } } if len(sessionNames) > 0 { if userConfig.Sync != nil && len(userConfig.Sync.Sessions) > 0 { userConfig.Sync.Sessions = sessionNames } } return environ.Sync(ctx, userConfig) }, }
View Source
var TransferHistoryCmd = &cobra.Command{ Use: "transfer-history", Short: "show transfer history", SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() configFile, err := cmd.Flags().GetString("config") if err != nil { return err } userConfig, err := bbgo.Load(configFile, false) if err != nil { return err } environ := bbgo.NewEnvironment() if err := BootstrapEnvironment(ctx, environ, userConfig); err != nil { return err } sessionName, err := cmd.Flags().GetString("session") if err != nil { return err } asset, err := cmd.Flags().GetString("asset") if err != nil { return err } session, ok := environ.Session(sessionName) if !ok { return fmt.Errorf("session %s not found", sessionName) } // default var now = time.Now() var since = now.AddDate(-1, 0, 0) var until = now sinceStr, err := cmd.Flags().GetString("since") if err != nil { return err } if len(sinceStr) > 0 { loc, err := time.LoadLocation("Asia/Taipei") if err != nil { return err } since, err = time.ParseInLocation("2006-01-02", sinceStr, loc) if err != nil { return err } } var records timeSlice exchange, ok := session.Exchange.(types.ExchangeTransferService) if !ok { return fmt.Errorf("exchange session %s does not implement transfer service", sessionName) } deposits, err := exchange.QueryDepositHistory(ctx, asset, since, until) if err != nil { return err } for _, d := range deposits { records = append(records, timeRecord{ Record: d, Time: d.EffectiveTime(), }) } withdraws, err := exchange.QueryWithdrawHistory(ctx, asset, since, until) if err != nil { return err } for _, w := range withdraws { records = append(records, timeRecord{ Record: w, Time: w.EffectiveTime(), }) } sort.Sort(records) for _, record := range records { switch record := record.Record.(type) { case types.Deposit: logrus.Infof("%s: <--- DEPOSIT %v %s [%s]", record.Time, record.Amount, record.Asset, record.Status) case types.Withdraw: logrus.Infof("%s: ---> WITHDRAW %v %s [%s]", record.ApplyTime, record.Amount, record.Asset, record.Status) default: logrus.Infof("unknown record: %+v", record) } } stats := calBaselineStats(asset, deposits, withdraws) for asset, quantity := range stats.TotalDeposit { logrus.Infof("total %s deposit: %v", asset, quantity) } for asset, quantity := range stats.TotalWithdraw { logrus.Infof("total %s withdraw: %v", asset, quantity) } for asset, quantity := range stats.BaselineBalance { logrus.Infof("baseline %s balance: %v", asset, quantity) } return nil }, }
Functions ¶
func BootstrapBacktestEnvironment ¶ added in v1.15.5
func BootstrapEnvironment ¶ added in v1.11.0
Types ¶
type BaselineStats ¶
type BaselineStats struct { Asset string TotalDeposit map[string]fixedpoint.Value TotalWithdraw map[string]fixedpoint.Value BaselineBalance map[string]fixedpoint.Value }
Source Files
¶
Click to show internal directories.
Click to hide internal directories.