From 4106443d427ae44fc2c112fcdbc4682b604a0c2b Mon Sep 17 00:00:00 2001 From: Max Resnick Date: Sun, 28 Jun 2026 09:51:02 -0400 Subject: [PATCH] Add Firebat Naxx self-play bot --- .../resources/Data/CardDefs-36393.xml | 31 +- SabberStoneCore/resources/Data/CardDefs.xml | 31 +- .../src/CardSets/FirebatNaxxCardsGen.cs | 122 ++ SabberStoneCore/src/CardSets/NaxxCardsGen.cs | 4 +- .../src/CardSets/Standard/Expert1CardsGen.cs | 5 +- .../src/CardSets/TavernBrawl/TbCardsGen.cs | 76 +- SabberStoneCore/src/Loader/CardDefs.cs | 1 + SabberStoneCore/src/Tasks/SpecificTask.cs | 5 +- .../SabberStoneCoreTest.csproj | 11 +- .../src/BasicAI/FirebatNaxxMatchupTest.cs | 628 ++++++++ .../SabberStoneBasicAI.csproj | 6 +- .../src/Bots/AlternatingTurnSearch.cs | 191 +++ .../src/Bots/BotInterfaces.cs | 126 ++ .../src/Bots/FirebatNaxxBenchmarkRunner.cs | 324 ++++ .../src/Bots/FirebatNaxxEvaluators.cs | 216 +++ .../src/Bots/FirebatNaxxGameFactory.cs | 65 + .../src/Bots/FirebatNaxxPolicies.cs | 406 +++++ .../src/Bots/FirebatNaxxTerminalViewer.cs | 229 +++ .../src/Bots/LinearPolicyValueNet.cs | 514 +++++++ .../src/Bots/MctsPolicyValueSearch.cs | 222 +++ .../src/Bots/PolicyValueModels.cs | 391 +++++ .../src/Bots/PolicyValueSearch.cs | 247 +++ .../src/Bots/RandomRolloutDatasetGenerator.cs | 540 +++++++ .../Bots/SelfPlayRolloutDatasetGenerator.cs | 234 +++ .../src/FirebatNaxxMatchupRunner.cs | 175 +++ .../SabberStoneBasicAI/src/Meta/DeckTypes.cs | 4 +- .../SabberStoneBasicAI/src/Meta/Decks.cs | 74 + .../SabberStoneBasicAI/src/Program.cs | 489 +++++- .../src/Score/ComboDruidScore.cs | 81 + .../src/Score/ZooWarlockScore.cs | 82 + tools/firebat_checkpoint_scheduler.py | 152 ++ tools/firebat_selfplay_dashboard.py | 1325 +++++++++++++++++ tools/train_firebat_value.py | 489 ++++++ 33 files changed, 7430 insertions(+), 66 deletions(-) create mode 100644 SabberStoneCore/src/CardSets/FirebatNaxxCardsGen.cs create mode 100644 SabberStoneCoreTest/src/BasicAI/FirebatNaxxMatchupTest.cs create mode 100644 core-extensions/SabberStoneBasicAI/src/Bots/AlternatingTurnSearch.cs create mode 100644 core-extensions/SabberStoneBasicAI/src/Bots/BotInterfaces.cs create mode 100644 core-extensions/SabberStoneBasicAI/src/Bots/FirebatNaxxBenchmarkRunner.cs create mode 100644 core-extensions/SabberStoneBasicAI/src/Bots/FirebatNaxxEvaluators.cs create mode 100644 core-extensions/SabberStoneBasicAI/src/Bots/FirebatNaxxGameFactory.cs create mode 100644 core-extensions/SabberStoneBasicAI/src/Bots/FirebatNaxxPolicies.cs create mode 100644 core-extensions/SabberStoneBasicAI/src/Bots/FirebatNaxxTerminalViewer.cs create mode 100644 core-extensions/SabberStoneBasicAI/src/Bots/LinearPolicyValueNet.cs create mode 100644 core-extensions/SabberStoneBasicAI/src/Bots/MctsPolicyValueSearch.cs create mode 100644 core-extensions/SabberStoneBasicAI/src/Bots/PolicyValueModels.cs create mode 100644 core-extensions/SabberStoneBasicAI/src/Bots/PolicyValueSearch.cs create mode 100644 core-extensions/SabberStoneBasicAI/src/Bots/RandomRolloutDatasetGenerator.cs create mode 100644 core-extensions/SabberStoneBasicAI/src/Bots/SelfPlayRolloutDatasetGenerator.cs create mode 100644 core-extensions/SabberStoneBasicAI/src/FirebatNaxxMatchupRunner.cs create mode 100644 core-extensions/SabberStoneBasicAI/src/Score/ComboDruidScore.cs create mode 100644 core-extensions/SabberStoneBasicAI/src/Score/ZooWarlockScore.cs create mode 100644 tools/firebat_checkpoint_scheduler.py create mode 100644 tools/firebat_selfplay_dashboard.py create mode 100644 tools/train_firebat_value.py diff --git a/SabberStoneCore/resources/Data/CardDefs-36393.xml b/SabberStoneCore/resources/Data/CardDefs-36393.xml index 78a65f436..8293ad962 100644 --- a/SabberStoneCore/resources/Data/CardDefs-36393.xml +++ b/SabberStoneCore/resources/Data/CardDefs-36393.xml @@ -155094,6 +155094,35 @@ o bien <b>Silencia</b> a un esbirro. + + + Moonfire + + + Deal 2 damage. + + + + + + + + + + + Dispel + + + <b>Silence</b> a minion. + + + + + + + + + 092a72e8-95cb-4208-8406-fa16c1866c2a @@ -434738,4 +434767,4 @@ de tu mazo. - \ No newline at end of file + diff --git a/SabberStoneCore/resources/Data/CardDefs.xml b/SabberStoneCore/resources/Data/CardDefs.xml index 53b3530f9..15256f673 100644 --- a/SabberStoneCore/resources/Data/CardDefs.xml +++ b/SabberStoneCore/resources/Data/CardDefs.xml @@ -161099,6 +161099,35 @@ o bien <b>Silencia</b> a un esbirro. + + + Moonfire + + + Deal 2 damage. + + + + + + + + + + + Dispel + + + <b>Silence</b> a minion. + + + + + + + + + Anregen @@ -423428,4 +423457,4 @@ de tu mazo. - \ No newline at end of file + diff --git a/SabberStoneCore/src/CardSets/FirebatNaxxCardsGen.cs b/SabberStoneCore/src/CardSets/FirebatNaxxCardsGen.cs new file mode 100644 index 000000000..40e2d813f --- /dev/null +++ b/SabberStoneCore/src/CardSets/FirebatNaxxCardsGen.cs @@ -0,0 +1,122 @@ +#region copyright +// SabberStone, Hearthstone Simulator in C# .NET Core +// Copyright (C) 2017-2019 SabberStone Team, darkfriend77 & rnilva +// +// SabberStone is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License. +// SabberStone is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +#endregion +using System.Collections.Generic; +using SabberStoneCore.Auras; +using SabberStoneCore.Conditions; +using SabberStoneCore.Enchants; +using SabberStoneCore.Enums; +using SabberStoneCore.Tasks; +using SabberStoneCore.Tasks.SimpleTasks; +using SabberStoneCore.Triggers; +using SabberStoneCore.src.Loader; + +namespace SabberStoneCore.CardSets +{ + /// + /// Minimal historical powers for the Firebat 2014 championship card ids used by the BasicAI matchup. + /// Tavern Brawl card definitions are intentionally not loaded globally in SabberStone. + /// + public static class FirebatNaxxCardsGen + { + public static void AddAll(IDictionary cards) + { + cards.Add("FB_Champs_EX1_165", new CardDef(new Power + { + PowerTask = new TransformTask("FB_Champs_OG_044a", EntityType.SOURCE) + })); + cards.Add("FB_Champs_EX1_165a", new CardDef(new Power + { + PowerTask = new TransformTask("FB_Champs_EX1_165t1", EntityType.SOURCE) + })); + cards.Add("FB_Champs_EX1_165b", new CardDef(new Power + { + PowerTask = new TransformTask("FB_Champs_EX1_165t2", EntityType.SOURCE) + })); + cards.Add("FB_Champs_EX1_166", new CardDef(new Dictionary { { PlayReq.REQ_TARGET_IF_AVAILABLE, 0 } }, new Power + { + })); + cards.Add("FB_Champs_EX1_166a", new CardDef(new Dictionary { { PlayReq.REQ_TARGET_TO_PLAY, 0 } }, new Power + { + PowerTask = new DamageTask(2, EntityType.TARGET) + })); + cards.Add("FB_Champs_EX1_166b", new CardDef(new Dictionary { { PlayReq.REQ_TARGET_TO_PLAY, 0 }, { PlayReq.REQ_MINION_TARGET, 0 } }, new Power + { + PowerTask = new SilenceTask(EntityType.TARGET) + })); + cards.Add("FB_Champs_EX1_tk9", new CardDef(new Power + { + Trigger = new Trigger(TriggerType.TURN_END) + { + EitherTurn = true, + SingleTask = new DestroyTask(EntityType.SOURCE) + } + })); + cards.Add("FB_Champs_EX1_169", new CardDef(new Power + { + PowerTask = new TempManaTask(2) + })); + cards.Add("FB_Champs_EX1_571", new CardDef(new Dictionary { { PlayReq.REQ_NUM_MINION_SLOTS, 1 } }, new Power + { + PowerTask = new SummonTask("FB_Champs_EX1_tk9", 3) + })); + cards.Add("FB_Champs_NEW1_008a", new CardDef(new Power + { + PowerTask = new DrawTask(2) + })); + cards.Add("FB_Champs_NEW1_008b", new CardDef(new Dictionary { { PlayReq.REQ_TARGET_TO_PLAY, 0 } }, new Power + { + PowerTask = new HealTask(5, EntityType.TARGET) + })); + + cards.Add("FB_Champs_CS2_188", new CardDef(new Dictionary { { PlayReq.REQ_TARGET_IF_AVAILABLE, 0 }, { PlayReq.REQ_MINION_TARGET, 0 } }, new Power + { + PowerTask = new AddEnchantmentTask("CS2_188o", EntityType.TARGET) + })); + cards.Add("FB_Champs_EX1_005", new CardDef(new Dictionary { { PlayReq.REQ_TARGET_IF_AVAILABLE, 0 }, { PlayReq.REQ_MINION_TARGET, 0 }, { PlayReq.REQ_TARGET_MIN_ATTACK, 7 } }, new Power + { + PowerTask = new DestroyTask(EntityType.TARGET) + })); + cards.Add("FB_Champs_EX1_029", new CardDef(new Power + { + DeathrattleTask = new DamageTask(2, EntityType.OP_HERO) + })); + cards.Add("FB_Champs_EX1_556", new CardDef(new Power + { + DeathrattleTask = new SummonTask("FB_Champs_skele21", SummonSide.DEATHRATTLE) + })); + cards.Add("FB_Champs_FP1_028e", new CardDef(new Power + { + Enchant = new OngoingEnchant(Effects.AttackHealth_N(1)) + })); + cards.Add("FB_Champs_FP1_028", new CardDef(new Power + { + InfoCardId = "FB_Champs_FP1_028e", + Trigger = new Trigger(TriggerType.SUMMON) + { + TriggerSource = TriggerSource.FRIENDLY, + Condition = SelfCondition.IsDeathrattleMinion, + SingleTask = new AddEnchantmentTask("FB_Champs_FP1_028e", EntityType.SOURCE) + } + })); + cards.Add("FB_Champs_NEW1_019", new CardDef(new Power + { + Trigger = new Trigger(TriggerType.AFTER_SUMMON) + { + TriggerSource = TriggerSource.MINIONS_EXCEPT_SELF, + SingleTask = ComplexTask.DamageRandomTargets(1, EntityType.ENEMIES, 1) + } + })); + } + } +} diff --git a/SabberStoneCore/src/CardSets/NaxxCardsGen.cs b/SabberStoneCore/src/CardSets/NaxxCardsGen.cs index af0fa4c2f..ef1420b97 100644 --- a/SabberStoneCore/src/CardSets/NaxxCardsGen.cs +++ b/SabberStoneCore/src/CardSets/NaxxCardsGen.cs @@ -565,7 +565,7 @@ private static void Neutral(IDictionary cards) // [FP1_028] Undertaker - COST:1 [ATK:1/HP:2] // - Set: naxx, Rarity: common // -------------------------------------------------------- - // Text: Whenever you summon a minion with Deathrattle, gain +1 Attack. + // Text: Whenever you summon a minion with Deathrattle, gain +1/+1. // -------------------------------------------------------- // RefTag: // - DEATHRATTLE = 1 @@ -650,7 +650,7 @@ private static void NeutralNonCollect(IDictionary cards) // -------------------------------------------------------- cards.Add("FP1_028e", new CardDef(new Power { - Enchant = new OngoingEnchant(Effects.Attack_N(1)) + Enchant = new OngoingEnchant(Effects.AttackHealth_N(1)) })); // ---------------------------------- ENCHANTMENT - NEUTRAL diff --git a/SabberStoneCore/src/CardSets/Standard/Expert1CardsGen.cs b/SabberStoneCore/src/CardSets/Standard/Expert1CardsGen.cs index 9a62faaca..b6325f171 100644 --- a/SabberStoneCore/src/CardSets/Standard/Expert1CardsGen.cs +++ b/SabberStoneCore/src/CardSets/Standard/Expert1CardsGen.cs @@ -296,14 +296,15 @@ private static void Druid(IDictionary cards) // [EX1_571] Force of Nature - COST:5 // - Fac: neutral, Set: expert1, Rarity: epic // -------------------------------------------------------- - // Text: Summon three 2/2 Treants. + // Text: Summon three 2/2 Treants with + // Charge that die at the end of the turn. // -------------------------------------------------------- // PlayReq: // - REQ_NUM_MINION_SLOTS = 1 // -------------------------------------------------------- cards.Add("EX1_571", new CardDef(new Dictionary() {{PlayReq.REQ_NUM_MINION_SLOTS,1}}, new Power { - PowerTask = new SummonTask("EX1_158t", 3) + PowerTask = new SummonTask("FB_Champs_EX1_tk9", 3) })); // ------------------------------------------ SPELL - DRUID diff --git a/SabberStoneCore/src/CardSets/TavernBrawl/TbCardsGen.cs b/SabberStoneCore/src/CardSets/TavernBrawl/TbCardsGen.cs index 2ffb00eea..dbcbb17ed 100644 --- a/SabberStoneCore/src/CardSets/TavernBrawl/TbCardsGen.cs +++ b/SabberStoneCore/src/CardSets/TavernBrawl/TbCardsGen.cs @@ -12,8 +12,13 @@ // GNU Affero General Public License for more details. #endregion using System.Collections.Generic; +using SabberStoneCore.Auras; +using SabberStoneCore.Conditions; using SabberStoneCore.Enchants; using SabberStoneCore.Enums; +using SabberStoneCore.Tasks; +using SabberStoneCore.Tasks.SimpleTasks; +using SabberStoneCore.Triggers; using SabberStoneCore.src.Loader; // ReSharper disable RedundantEmptyObjectOrCollectionInitializer @@ -3513,9 +3518,7 @@ private static void DruidNonCollect(IDictionary cards) // -------------------------------------------------------- cards.Add("FB_Champs_EX1_165", new CardDef(new Power { - // TODO [FB_Champs_EX1_165] Druid of the Claw && Test: Druid of the Claw_FB_Champs_EX1_165 - //PowerTask = null, - //Trigger = null, + PowerTask = new TransformTask("FB_Champs_OG_044a", EntityType.SOURCE) })); // ----------------------------------------- MINION - DRUID @@ -3582,9 +3585,10 @@ private static void DruidNonCollect(IDictionary cards) // -------------------------------------------------------- cards.Add("FB_Champs_EX1_tk9", new CardDef(new Power { - // TODO [FB_Champs_EX1_tk9] Treant && Test: Treant_FB_Champs_EX1_tk9 - //PowerTask = null, - //Trigger = null, + Trigger = new Trigger(TriggerType.TURN_END) + { + SingleTask = new DestroyTask(EntityType.SOURCE) + } })); // ----------------------------------------- MINION - DRUID @@ -3661,9 +3665,7 @@ private static void DruidNonCollect(IDictionary cards) // -------------------------------------------------------- cards.Add("FB_Champs_EX1_165a", new CardDef(new Power { - // TODO [FB_Champs_EX1_165a] Cat Form && Test: Cat Form_FB_Champs_EX1_165a - //PowerTask = null, - //Trigger = null, + PowerTask = new TransformTask("FB_Champs_EX1_165t1", EntityType.SOURCE) })); // ------------------------------------------ SPELL - DRUID @@ -3674,9 +3676,7 @@ private static void DruidNonCollect(IDictionary cards) // -------------------------------------------------------- cards.Add("FB_Champs_EX1_165b", new CardDef(new Power { - // TODO [FB_Champs_EX1_165b] Bear Form && Test: Bear Form_FB_Champs_EX1_165b - //PowerTask = null, - //Trigger = null, + PowerTask = new TransformTask("FB_Champs_EX1_165t2", EntityType.SOURCE) })); // ------------------------------------------ SPELL - DRUID @@ -3687,9 +3687,7 @@ private static void DruidNonCollect(IDictionary cards) // -------------------------------------------------------- cards.Add("FB_Champs_EX1_169", new CardDef(new Power { - // TODO [FB_Champs_EX1_169] Innervate && Test: Innervate_FB_Champs_EX1_169 - //PowerTask = null, - //Trigger = null, + PowerTask = new TempManaTask(2) })); // ------------------------------------------ SPELL - DRUID @@ -3704,9 +3702,7 @@ private static void DruidNonCollect(IDictionary cards) // -------------------------------------------------------- cards.Add("FB_Champs_EX1_571", new CardDef(new Dictionary() {{PlayReq.REQ_NUM_MINION_SLOTS,1}}, new Power { - // TODO [FB_Champs_EX1_571] Force of Nature && Test: Force of Nature_FB_Champs_EX1_571 - //PowerTask = null, - //Trigger = null, + PowerTask = new SummonTask("FB_Champs_EX1_tk9", 3) })); // ------------------------------------------ SPELL - DRUID @@ -3717,9 +3713,7 @@ private static void DruidNonCollect(IDictionary cards) // -------------------------------------------------------- cards.Add("FB_Champs_NEW1_008a", new CardDef(new Power { - // TODO [FB_Champs_NEW1_008a] Ancient Teachings && Test: Ancient Teachings_FB_Champs_NEW1_008a - //PowerTask = null, - //Trigger = null, + PowerTask = new DrawTask(2) })); // ------------------------------------------ SPELL - DRUID @@ -3733,9 +3727,7 @@ private static void DruidNonCollect(IDictionary cards) // -------------------------------------------------------- cards.Add("FB_Champs_NEW1_008b", new CardDef(new Dictionary() {{PlayReq.REQ_TARGET_TO_PLAY,0}}, new Power { - // TODO [FB_Champs_NEW1_008b] Ancient Secrets && Test: Ancient Secrets_FB_Champs_NEW1_008b - //PowerTask = null, - //Trigger = null, + PowerTask = new HealTask(5, EntityType.TARGET) })); // ------------------------------------------ SPELL - DRUID @@ -6989,8 +6981,7 @@ private static void NeutralNonCollect(IDictionary cards) // -------------------------------------------------------- cards.Add("FB_Champs_FP1_028e", new CardDef(new Power { - // TODO [FB_Champs_FP1_028e] Darkness Calls && Test: Darkness Calls_FB_Champs_FP1_028e - //Enchant = Enchants.Enchants.GetAutoEnchantFromText("FB_Champs_FP1_028e") + Enchant = new OngoingEnchant(Effects.AttackHealth_N(1)) })); // ---------------------------------- ENCHANTMENT - NEUTRAL @@ -9645,9 +9636,7 @@ private static void NeutralNonCollect(IDictionary cards) // -------------------------------------------------------- cards.Add("FB_Champs_CS2_188", new CardDef(new Dictionary() {{PlayReq.REQ_TARGET_IF_AVAILABLE,0},{PlayReq.REQ_MINION_TARGET,0}}, new Power { - // TODO [FB_Champs_CS2_188] Abusive Sergeant && Test: Abusive Sergeant_FB_Champs_CS2_188 - //PowerTask = null, - //Trigger = null, + PowerTask = new AddEnchantmentTask("CS2_188o", EntityType.TARGET) })); // --------------------------------------- MINION - NEUTRAL @@ -9666,9 +9655,7 @@ private static void NeutralNonCollect(IDictionary cards) // -------------------------------------------------------- cards.Add("FB_Champs_EX1_005", new CardDef(new Dictionary() {{PlayReq.REQ_TARGET_IF_AVAILABLE,0},{PlayReq.REQ_MINION_TARGET,0},{PlayReq.REQ_TARGET_MIN_ATTACK,7}}, new Power { - // TODO [FB_Champs_EX1_005] Big Game Hunter && Test: Big Game Hunter_FB_Champs_EX1_005 - //PowerTask = null, - //Trigger = null, + PowerTask = new DestroyTask(EntityType.TARGET) })); // --------------------------------------- MINION - NEUTRAL @@ -9682,9 +9669,7 @@ private static void NeutralNonCollect(IDictionary cards) // -------------------------------------------------------- cards.Add("FB_Champs_EX1_029", new CardDef(new Power { - // TODO [FB_Champs_EX1_029] Leper Gnome && Test: Leper Gnome_FB_Champs_EX1_029 - //PowerTask = null, - //Trigger = null, + DeathrattleTask = new DamageTask(2, EntityType.OP_HERO) })); // --------------------------------------- MINION - NEUTRAL @@ -9711,9 +9696,7 @@ private static void NeutralNonCollect(IDictionary cards) // -------------------------------------------------------- cards.Add("FB_Champs_EX1_556", new CardDef(new Power { - // TODO [FB_Champs_EX1_556] Harvest Golem && Test: Harvest Golem_FB_Champs_EX1_556 - //PowerTask = null, - //Trigger = null, + DeathrattleTask = new SummonTask("FB_Champs_skele21", SummonSide.DEATHRATTLE) })); // --------------------------------------- MINION - NEUTRAL @@ -9727,10 +9710,13 @@ private static void NeutralNonCollect(IDictionary cards) // -------------------------------------------------------- cards.Add("FB_Champs_FP1_028", new CardDef(new Power { - // TODO [FB_Champs_FP1_028] Undertaker && Test: Undertaker_FB_Champs_FP1_028 InfoCardId = "FB_Champs_FP1_028e", - //PowerTask = null, - //Trigger = null, + Trigger = new Trigger(TriggerType.SUMMON) + { + TriggerSource = TriggerSource.FRIENDLY, + Condition = SelfCondition.IsDeathrattleMinion, + SingleTask = new AddEnchantmentTask("FB_Champs_FP1_028e", EntityType.SOURCE) + } })); // --------------------------------------- MINION - NEUTRAL @@ -9834,9 +9820,11 @@ private static void NeutralNonCollect(IDictionary cards) // -------------------------------------------------------- cards.Add("FB_Champs_NEW1_019", new CardDef(new Power { - // TODO [FB_Champs_NEW1_019] Knife Juggler && Test: Knife Juggler_FB_Champs_NEW1_019 - //PowerTask = null, - //Trigger = null, + Trigger = new Trigger(TriggerType.AFTER_SUMMON) + { + TriggerSource = TriggerSource.MINIONS_EXCEPT_SELF, + SingleTask = ComplexTask.DamageRandomTargets(1, EntityType.ENEMIES, 1) + } })); // --------------------------------------- MINION - NEUTRAL diff --git a/SabberStoneCore/src/Loader/CardDefs.cs b/SabberStoneCore/src/Loader/CardDefs.cs index 70b0a361b..9eb9ab966 100644 --- a/SabberStoneCore/src/Loader/CardDefs.cs +++ b/SabberStoneCore/src/Loader/CardDefs.cs @@ -77,6 +77,7 @@ private CardDefs() LootapaloozaCardsGen.AddAll(_cardDefsDic); HofCardsGen.AddAll(_cardDefsDic); + FirebatNaxxCardsGen.AddAll(_cardDefsDic); // Tavern Brawl //TbCardsGen.AddAll(_powerDic); diff --git a/SabberStoneCore/src/Tasks/SpecificTask.cs b/SabberStoneCore/src/Tasks/SpecificTask.cs index d6776d77d..52ef40d22 100644 --- a/SabberStoneCore/src/Tasks/SpecificTask.cs +++ b/SabberStoneCore/src/Tasks/SpecificTask.cs @@ -604,7 +604,7 @@ public static ISimpleTask TessGreymane IList playedCards = c.PlayHistory .Select(e => e.SourceCard) .Where(card => card.Class != CardClass.NEUTRAL && card.Class != c.Hero.Card.Class) - .ToArray() + .ToList() .Shuffle(g.Random); foreach (Card card in playedCards) @@ -667,7 +667,7 @@ public static ISimpleTask Shudderwock IList playedCards = c.PlayHistory .Select(e => e.SourceCard) .Where(card => card[GameTag.BATTLECRY] == 1 && card.AssetId != 48111) - .ToArray() + .ToList() .Shuffle(game.Random); int count = 0; @@ -1184,4 +1184,3 @@ public override TaskState Process(in Game game, in Controller controller, in IEn } } - diff --git a/SabberStoneCoreTest/SabberStoneCoreTest.csproj b/SabberStoneCoreTest/SabberStoneCoreTest.csproj index 7fbf78e2a..c4ebc6ca2 100644 --- a/SabberStoneCoreTest/SabberStoneCoreTest.csproj +++ b/SabberStoneCoreTest/SabberStoneCoreTest.csproj @@ -1,7 +1,7 @@  - netcoreapp2.0 + netcoreapp2.0;net10.0 SabberStoneCore.Test latest Debug;Release;NoSpan @@ -25,15 +25,22 @@ 1701;1702;1705;xUnit1004 - + + + + + + + + diff --git a/SabberStoneCoreTest/src/BasicAI/FirebatNaxxMatchupTest.cs b/SabberStoneCoreTest/src/BasicAI/FirebatNaxxMatchupTest.cs new file mode 100644 index 000000000..84ecff932 --- /dev/null +++ b/SabberStoneCoreTest/src/BasicAI/FirebatNaxxMatchupTest.cs @@ -0,0 +1,628 @@ +#region copyright +// SabberStone, Hearthstone Simulator in C# .NET Core +// Copyright (C) 2017-2019 SabberStone Team, darkfriend77 & rnilva +// +// SabberStone is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License. +// SabberStone is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +#endregion +using System.IO; +using System.Linq; +using System.Text; +using SabberStoneBasicAI.Bots; +using SabberStoneBasicAI.Meta; +using SabberStoneBasicAI.Score; +using SabberStoneCore.Config; +using SabberStoneCore.Enums; +using SabberStoneCore.Model; +using SabberStoneCore.Model.Entities; +using SabberStoneCore.Tasks.PlayerTasks; +using SabberStoneCoreTest; +using Xunit; +using Generic = SabberStoneCore.Actions.Generic; + +namespace SabberStoneCoreTest.BasicAI +{ + public class FirebatNaxxMatchupTest + { + [Fact] + public void FirebatDecksHaveThirtyKnownCards() + { + Assert.Equal(30, Decks.FirebatNaxxComboDruid.Count); + Assert.Equal(30, Decks.FirebatNaxxZooWarlock.Count); + Assert.DoesNotContain(null, Decks.FirebatNaxxComboDruid); + Assert.DoesNotContain(null, Decks.FirebatNaxxZooWarlock); + } + + [Fact] + public void OldForceOfNatureSummonsTemporaryChargeTreants() + { + Game game = CreateDruidGame(); + IPlayable force = Generic.DrawCard(game.CurrentPlayer, Cards.FromId("EX1_571")); + force.Cost = 0; + + Assert.True(game.Process(PlayCardTask.Spell(game.CurrentPlayer, force))); + + Assert.Equal(3, game.Player1.BoardZone.Count); + Assert.All(game.Player1.BoardZone, minion => + { + Assert.Equal("FB_Champs_EX1_tk9", minion.Card.Id); + Assert.True(minion.HasCharge); + }); + + game.EndTurn(); + + Assert.Empty(game.Player1.BoardZone); + } + + [Fact] + public void OldUndertakerGainsAttackAndHealthFromDeathrattleSummon() + { + Game game = CreateWarlockGame(); + Minion undertaker = (Minion)Generic.DrawCard(game.CurrentPlayer, Cards.FromId("FB_Champs_FP1_028")); + Minion creeper = (Minion)Generic.DrawCard(game.CurrentPlayer, Cards.FromId("FP1_002")); + + game.ProcessCard(undertaker, asZeroCost: true); + game.ProcessCard(creeper, asZeroCost: true); + + Assert.Equal(2, undertaker.AttackDamage); + Assert.Equal(3, undertaker.Health); + } + + [Fact] + public void OldAncientOfLoreDrawsTwoCards() + { + Game game = CreateDruidGame(); + IPlayable lore = Generic.DrawCard(game.CurrentPlayer, Cards.FromId("FB_Champs_NEW1_008")); + int handCountBefore = game.CurrentPlayer.HandZone.Count; + + game.ProcessCard(lore, asZeroCost: true, chooseOne: 1); + + Assert.Equal(handCountBefore + 1, game.CurrentPlayer.HandZone.Count); + } + + [Fact] + public void ChampionshipKeeperUsesOldBodyAndKeeperOptions() + { + Game game = CreateDruidGame(); + IPlayable firstKeeper = Generic.DrawCard(game.CurrentPlayer, Cards.FromId("FB_Champs_EX1_166")); + IPlayable mark = Generic.DrawCard(game.CurrentPlayer, Cards.FromId("CS2_009")); + IPlayable secondKeeper = Generic.DrawCard(game.CurrentPlayer, Cards.FromId("FB_Champs_EX1_166")); + + Assert.Equal(2, firstKeeper.Card.ATK); + Assert.Equal(4, firstKeeper.Card.Health); + Assert.Equal("FB_Champs_EX1_166a", firstKeeper.ChooseOnePlayables[0].Card.Id); + Assert.Equal("FB_Champs_EX1_166b", firstKeeper.ChooseOnePlayables[1].Card.Id); + + game.ProcessCard(firstKeeper, game.CurrentOpponent.Hero, asZeroCost: true, chooseOne: 1); + Assert.Equal(28, game.CurrentOpponent.Hero.Health); + + game.ProcessCard(mark, game.CurrentPlayer.BoardZone[0], asZeroCost: true); + Assert.Equal(6, game.CurrentPlayer.BoardZone[0].Health); + + game.ProcessCard(secondKeeper, game.CurrentPlayer.BoardZone[0], asZeroCost: true, chooseOne: 2); + Assert.Equal(4, game.CurrentPlayer.BoardZone[0].Health); + } + + [Fact] + public void CandidateMirrorCompletesDeterministicSmokeGame() + { + var config = new SearchConfig + { + MaxDepth = 3, + MaxWidth = 20, + CandidateLines = 4, + OpponentDepth = 2, + OpponentResponseLines = 3, + MctsIterations = 20, + TimeBudgetMs = 500 + }; + + FirebatNaxxScenarioResult scenario = FirebatNaxxBenchmarkRunner.RunCandidateMirror(1, 1, config); + + Assert.Single(scenario.Games); + Assert.Equal(0, scenario.Errors); + Assert.Equal(0, scenario.Unfinished); + Assert.True(scenario.Player1Wins + scenario.Player2Wins + scenario.Ties == 1); + Assert.True(scenario.AverageMoveMs < 500); + } + + [Fact] + public void FirebatMulliganPoliciesReturnCardsToKeep() + { + Game druidGame = CreateDruidGame(); + IPlayable innervate = Generic.DrawCard(druidGame.CurrentPlayer, Cards.FromId("FB_Champs_EX1_169")); + IPlayable wrath = Generic.DrawCard(druidGame.CurrentPlayer, Cards.FromId("EX1_154")); + IPlayable cenarius = Generic.DrawCard(druidGame.CurrentPlayer, Cards.FromId("EX1_573")); + + var druidKeep = new ComboDruidScore().MulliganRule().Invoke(new[] { innervate, wrath, cenarius }.ToList()); + + Assert.Contains(innervate.Id, druidKeep); + Assert.Contains(wrath.Id, druidKeep); + Assert.DoesNotContain(cenarius.Id, druidKeep); + + Game warlockGame = CreateWarlockGame(); + IPlayable flameImp = Generic.DrawCard(warlockGame.CurrentPlayer, Cards.FromId("EX1_319")); + IPlayable undertaker = Generic.DrawCard(warlockGame.CurrentPlayer, Cards.FromId("FB_Champs_FP1_028")); + IPlayable soulfire = Generic.DrawCard(warlockGame.CurrentPlayer, Cards.FromId("EX1_308")); + + var zooKeep = new ZooWarlockScore().MulliganRule().Invoke(new[] { flameImp, undertaker, soulfire }.ToList()); + + Assert.Contains(flameImp.Id, zooKeep); + Assert.Contains(undertaker.Id, zooKeep); + Assert.DoesNotContain(soulfire.Id, zooKeep); + } + + [Fact] + public void RandomPolicyDoesNotEndTurnWhenActionCapLeavesPlayableActions() + { + Game game = CreateDruidGame(); + var policy = new RandomActionPolicy("test-random", seed: 19, maxActionsPerTurn: 1); + + Assert.Contains(game.CurrentPlayer.Options(), task => !(task is EndTurnTask) && !(task is ConcedeTask)); + + BotTurnDecision decision = policy.SelectTurn(game, game.CurrentPlayer, new BotTurnContext(new SearchConfig(), null)); + + Assert.DoesNotContain(decision.Tasks, task => task is EndTurnTask); + Assert.Single(decision.Tasks); + } + + [Fact] + public void HiddenStatePredictorBuildsBeliefFromPublicInformation() + { + Game game = FirebatNaxxGameFactory.Create(7); + game.StartGame(); + + var predictor = new FirebatNaxxHiddenStatePredictor(sampleCount: 4); + HiddenStateBelief belief = predictor.Predict(game, game.Player1.Id); + + Assert.Equal(game.Player2.HandZone.Count, belief.ObservedHandSize); + Assert.Equal(30, belief.UnknownCounts.Values.Sum()); + Assert.Equal(4, belief.Samples.Count); + Assert.Equal(belief.ObservedHandSize, belief.ExpectedHandCounts.Values.Sum(), 3); + Assert.All(belief.UnknownCounts.Keys, cardId => Assert.Contains(Decks.FirebatNaxxZooWarlock, card => card.Id == cardId)); + } + + [Fact] + public void PolicyValueNetsProduceLegalAndFullInformationPredictions() + { + Game game = FirebatNaxxGameFactory.Create(11); + game.StartGame(); + game.Process(ChooseTask.Mulligan(game.Player1, new System.Collections.Generic.List())); + game.Process(ChooseTask.Mulligan(game.Player2, new System.Collections.Generic.List())); + game.MainReady(); + + FirebatNaxxSide side = game.CurrentPlayer.HeroClass == CardClass.DRUID + ? FirebatNaxxSide.ComboDruid + : FirebatNaxxSide.ZooWarlock; + var legalNet = new FirebatNaxxPolicyValueNet(side); + var fullInfoNet = new FirebatNaxxPolicyValueNet(side, fullInformation: true); + var legalActions = game.CurrentPlayer.Options().Where(task => !(task is ConcedeTask)).ToList(); + + PolicyValuePrediction legal = legalNet.Predict(game, game.CurrentPlayer.Id, legalActions); + PolicyValuePrediction fullInfo = fullInfoNet.Predict(game, game.CurrentPlayer.Id, legalActions); + + Assert.InRange(legal.Value, -1.0, 1.0); + Assert.InRange(fullInfo.Value, -1.0, 1.0); + Assert.NotEmpty(legal.ActionPriors); + Assert.Contains("belief_expected_opponent_reach", legal.Features.Keys); + Assert.Contains("full_info_opponent_reach", fullInfo.Features.Keys); + } + + [Fact] + public void LinearPolicyValueNetLoadsExportedWeights() + { + string path = Path.Combine(Path.GetTempPath(), "firebat-linear-value-" + Path.GetRandomFileName() + ".json"); + try + { + File.WriteAllText(path, BuildLinearModelJson("legal_value", "combo_druid", 0.25)); + Game game = CreateDruidGame(); + var net = new LinearPolicyValueNet(path, FirebatNaxxSide.ComboDruid); + PolicyValuePrediction prediction = net.Predict(game, game.CurrentPlayer.Id, game.CurrentPlayer.Options().Where(task => !(task is ConcedeTask)).ToList()); + + Assert.InRange(prediction.Value, 0.20, 0.30); + Assert.NotEmpty(prediction.ActionPriors); + Assert.Contains("bias", prediction.Features.Keys); + } + finally + { + if (File.Exists(path)) + File.Delete(path); + } + } + + [Fact] + public void LinearPolicyValueNetUsesExportedPolicyHead() + { + string path = Path.Combine(Path.GetTempPath(), "firebat-linear-policy-value-" + Path.GetRandomFileName() + ".json"); + try + { + File.WriteAllText(path, BuildLinearPolicyModelJson()); + Game game = CreateDruidGame(); + var net = new LinearPolicyValueNet(path, FirebatNaxxSide.ComboDruid); + var legalActions = game.CurrentPlayer.Options().Where(task => !(task is ConcedeTask)).ToList(); + + Assert.Contains(legalActions, task => task is PlayCardTask); + Assert.Contains(legalActions, task => task is EndTurnTask); + + PolicyValuePrediction prediction = net.Predict(game, game.CurrentPlayer.Id, legalActions); + double playCardPrior = legalActions + .Where(task => task is PlayCardTask) + .Sum(task => prediction.ActionPriors[FirebatNaxxPolicyValueNet.ActionKey(task)]); + double endTurnPrior = legalActions + .Where(task => task is EndTurnTask) + .Sum(task => prediction.ActionPriors[FirebatNaxxPolicyValueNet.ActionKey(task)]); + + Assert.True(playCardPrior > endTurnPrior); + } + finally + { + if (File.Exists(path)) + File.Delete(path); + } + } + + [Fact] + public void PolicyValueMirrorCompletesDeterministicSmokeGame() + { + var config = new SearchConfig + { + MaxDepth = 3, + MaxWidth = 20, + CandidateLines = 4, + OpponentDepth = 2, + OpponentResponseLines = 3, + MctsIterations = 20, + TimeBudgetMs = 500 + }; + + FirebatNaxxScenarioResult scenario = FirebatNaxxBenchmarkRunner.RunPolicyValueMirror(1, 2, config); + + Assert.Single(scenario.Games); + Assert.Equal(0, scenario.Errors); + Assert.Equal(0, scenario.Unfinished); + Assert.True(scenario.Player1Wins + scenario.Player2Wins + scenario.Ties == 1); + Assert.True(scenario.AverageMoveMs < 500); + } + + [Fact] + public void PolicyValueSearchProducesRootActionTargetDistribution() + { + Game game = CreateDruidGame(); + var policy = FirebatNaxxPolicies.PolicyValueDruid(); + BotTurnDecision decision = policy.SelectTurn(game, game.CurrentPlayer, new BotTurnContext(new SearchConfig + { + MaxDepth = 3, + MaxWidth = 20, + CandidateLines = 6, + OpponentDepth = 2, + OpponentResponseLines = 3, + MctsIterations = 24, + TimeBudgetMs = 500 + }, FirebatNaxxPolicies.PolicyValueZoo())); + + Assert.NotEmpty(decision.Tasks); + Assert.Equal("mcts-policy-value-search", decision.SearchName); + Assert.NotEmpty(decision.RootActionTargets); + Assert.Contains(FirebatNaxxPolicyValueNet.ActionKey(decision.Tasks[0]), decision.RootActionTargets.Keys); + Assert.InRange(decision.RootActionTargets.Values.Sum(), 0.99, 1.01); + Assert.All(decision.RootActionTargets.Values, value => Assert.InRange(value, 0.0, 1.0)); + } + + [Fact] + public void RandomRolloutGeneratorWritesDeterministicTrainingRows() + { + string firstPath = Path.Combine(Path.GetTempPath(), "firebat-rollouts-a-" + Path.GetRandomFileName() + ".jsonl"); + string secondPath = Path.Combine(Path.GetTempPath(), "firebat-rollouts-b-" + Path.GetRandomFileName() + ".jsonl"); + try + { + var config = new RandomRolloutDatasetConfig + { + Games = 1, + Seed = 17, + MaxTurns = 20, + MaxActionsPerTurn = 50, + OutputPath = firstPath + }; + RandomRolloutDatasetResult first = RandomRolloutDatasetGenerator.Generate(config); + config.OutputPath = secondPath; + RandomRolloutDatasetResult second = RandomRolloutDatasetGenerator.Generate(config); + + string firstOutput = File.ReadAllText(firstPath); + string secondOutput = File.ReadAllText(secondPath); + string[] lines = File.ReadAllLines(firstPath); + + Assert.Equal(firstOutput, secondOutput); + Assert.True(first.RowsWritten > 0); + Assert.Equal(first.RowsWritten, second.RowsWritten); + Assert.Equal(first.RowsWritten, lines.Length); + Assert.Equal(first.LegalValueRows, first.FullInfoValueRows); + Assert.Equal(first.LegalValueRows, first.HiddenStateRows); + Assert.Contains(lines, line => line.Contains("\"kind\":\"legal_value\"")); + Assert.Contains(lines, line => line.Contains("\"kind\":\"full_info_value\"")); + Assert.Contains(lines, line => line.Contains("\"kind\":\"hidden_state\"")); + Assert.Contains(lines, line => line.Contains("\"value\":")); + Assert.Contains(lines, line => line.Contains("\"target\":{\"hiddenPlayerId\"")); + } + finally + { + if (File.Exists(firstPath)) + File.Delete(firstPath); + if (File.Exists(secondPath)) + File.Delete(secondPath); + } + } + + [Fact] + public void SelfPlayRolloutGeneratorWritesTrainingRows() + { + string firstPath = Path.Combine(Path.GetTempPath(), "firebat-selfplay-a-" + Path.GetRandomFileName() + ".jsonl"); + string secondPath = Path.Combine(Path.GetTempPath(), "firebat-selfplay-b-" + Path.GetRandomFileName() + ".jsonl"); + try + { + var config = new SelfPlayRolloutDatasetConfig + { + Games = 1, + Seed = 23, + MaxTurns = 30, + MaxDecisions = 120, + OutputPath = firstPath, + Policy = "baseline", + SearchConfig = new SearchConfig + { + MaxDepth = 3, + MaxWidth = 20, + CandidateLines = 4, + OpponentDepth = 2, + OpponentResponseLines = 3, + TimeBudgetMs = 500 + } + }; + SelfPlayRolloutDatasetResult first = SelfPlayRolloutDatasetGenerator.Generate(config); + config.OutputPath = secondPath; + SelfPlayRolloutDatasetResult second = SelfPlayRolloutDatasetGenerator.Generate(config); + + string[] lines = File.ReadAllLines(firstPath); + + Assert.Equal(0, first.Errors); + Assert.Equal(0, second.Errors); + Assert.Equal(1, first.CompletedGames + first.UnfinishedGames); + Assert.Equal(1, second.CompletedGames + second.UnfinishedGames); + Assert.True(first.Decisions > 0); + Assert.True(second.Decisions > 0); + Assert.True(first.ActionsPlayed > 0); + Assert.True(second.ActionsPlayed > 0); + Assert.True(first.RowsWritten > 0); + Assert.True(second.RowsWritten > 0); + Assert.Equal(first.RowsWritten, lines.Length); + Assert.Equal(first.LegalValueRows, first.FullInfoValueRows); + Assert.Equal(first.LegalValueRows, first.HiddenStateRows); + Assert.Contains(lines, line => line.Contains("\"policy\":\"selfplay-baseline\"")); + Assert.Contains(lines, line => line.Contains("\"kind\":\"policy\"")); + Assert.Contains(lines, line => line.Contains("\"kind\":\"legal_value\"")); + Assert.Contains(lines, line => line.Contains("\"kind\":\"full_info_value\"")); + Assert.Contains(lines, line => line.Contains("\"kind\":\"hidden_state\"")); + Assert.True(first.PolicyRows > 0); + } + finally + { + if (File.Exists(firstPath)) + File.Delete(firstPath); + if (File.Exists(secondPath)) + File.Delete(secondPath); + } + } + + [Fact] + public void TerminalViewerPrintsAsciiGameState() + { + var writer = new StringWriter(); + var result = FirebatNaxxTerminalViewer.Show(new FirebatNaxxTerminalViewerConfig + { + Seed = 31, + Policy = "baseline", + MaxTurns = 1, + MaxDecisions = 1, + SearchConfig = new SearchConfig + { + MaxDepth = 2, + MaxWidth = 10, + CandidateLines = 2, + OpponentDepth = 1, + OpponentResponseLines = 1, + TimeBudgetMs = 200 + } + }, writer); + + string output = writer.ToString(); + + Assert.Null(result.Error); + Assert.Equal(1, result.Decisions); + Assert.Contains("Firebat Naxx game viewer", output); + Assert.Contains("--- Initial state ---", output); + Assert.Contains("+-Firebat", output); + Assert.Contains("decision score=", output); + Assert.Contains("Result:", output); + } + + [Fact] + public void RandomPolicyViewerPrintsTrainedEvaluationSpot() + { + string path = Path.Combine(Path.GetTempPath(), "firebat-linear-value-bundle-" + Path.GetRandomFileName() + ".json"); + try + { + File.WriteAllText(path, BuildLinearModelBundleJson(0.25)); + var writer = new StringWriter(); + FirebatNaxxTerminalViewerResult result = FirebatNaxxTerminalViewer.Show(new FirebatNaxxTerminalViewerConfig + { + Seed = 37, + Policy = "random", + EvaluationDecision = 2, + EvaluationPolicy = "trained-pv", + EvaluationWeightsPath = path, + MaxTurns = 5, + MaxDecisions = 5 + }, writer); + + string output = writer.ToString(); + + Assert.Null(result.Error); + Assert.Equal(1, result.Decisions); + Assert.Contains("policy=viewer-random", output); + Assert.Contains("evalPolicy=eval-trained-policy-value", output); + Assert.Contains("--- Evaluation spot before decision 2 ---", output); + Assert.Contains("linear-legal_value-combo_druid-policy-value-net evaluator", output); + Assert.Contains("linear-legal_value-zoo_warlock-policy-value-net evaluator", output); + Assert.Contains("value=0.245", output); + } + finally + { + if (File.Exists(path)) + File.Delete(path); + } + } + + [Fact] + public void RandomPolicyViewerPrintsTrainedFullInformationEvaluationSpot() + { + string path = Path.Combine(Path.GetTempPath(), "firebat-linear-value-full-info-bundle-" + Path.GetRandomFileName() + ".json"); + try + { + File.WriteAllText(path, BuildLinearModelBundleJson(0.25)); + var writer = new StringWriter(); + FirebatNaxxTerminalViewerResult result = FirebatNaxxTerminalViewer.Show(new FirebatNaxxTerminalViewerConfig + { + Seed = 37, + Policy = "random", + EvaluationDecision = 2, + EvaluationPolicy = "trained-full-info-pv", + EvaluationWeightsPath = path, + MaxTurns = 5, + MaxDecisions = 5 + }, writer); + + string output = writer.ToString(); + + Assert.Null(result.Error); + Assert.Contains("evalPolicy=eval-trained-full-info-policy-value", output); + Assert.Contains("linear-full_info_value-combo_druid-policy-value-net evaluator", output); + Assert.Contains("linear-full_info_value-zoo_warlock-policy-value-net evaluator", output); + Assert.Contains("value=0.245", output); + } + finally + { + if (File.Exists(path)) + File.Delete(path); + } + } + + private static Game CreateDruidGame() + { + return CreateGame(CardClass.DRUID); + } + + private static Game CreateWarlockGame() + { + return CreateGame(CardClass.WARLOCK); + } + + private static Game CreateGame(CardClass playerClass) + { + var game = new Game(new GameConfig + { + StartPlayer = 1, + Player1HeroClass = playerClass, + Player2HeroClass = CardClass.WARLOCK, + FillDecks = true, + FillDecksPredictably = true + }); + game.StartGame(); + game.Player1.BaseMana = 10; + game.Player2.BaseMana = 10; + return game; + } + + private static string BuildLinearModelJson(string kind, string side, double biasWeight) + { + var sb = new StringBuilder(); + sb.Append("{\"schema\":\"firebat_naxx_linear_value_v1\",\"featureSchema\":["); + for (int i = 0; i < FirebatNaxxFeatureExtractor.FeatureSchema.Count; i++) + { + if (i > 0) + sb.Append(','); + sb.Append('"').Append(FirebatNaxxFeatureExtractor.FeatureSchema[i]).Append('"'); + } + sb.Append("],\"models\":[{\"kind\":\"").Append(kind).Append("\",\"side\":\"").Append(side).Append("\",\"weights\":["); + for (int i = 0; i < FirebatNaxxFeatureExtractor.FeatureSchema.Count; i++) + { + if (i > 0) + sb.Append(','); + sb.Append(i == 0 ? biasWeight.ToString(System.Globalization.CultureInfo.InvariantCulture) : "0"); + } + sb.Append("]}]}"); + return sb.ToString(); + } + + private static string BuildLinearModelBundleJson(double biasWeight) + { + var sb = new StringBuilder(); + sb.Append("{\"schema\":\"firebat_naxx_linear_value_v1\",\"featureSchema\":["); + for (int i = 0; i < FirebatNaxxFeatureExtractor.FeatureSchema.Count; i++) + { + if (i > 0) + sb.Append(','); + sb.Append('"').Append(FirebatNaxxFeatureExtractor.FeatureSchema[i]).Append('"'); + } + sb.Append("],\"models\":["); + string[] kinds = { "legal_value", "full_info_value" }; + string[] sides = { "combo_druid", "zoo_warlock" }; + bool first = true; + foreach (string kind in kinds) + { + foreach (string side in sides) + { + if (!first) + sb.Append(','); + first = false; + sb.Append("{\"kind\":\"").Append(kind).Append("\",\"side\":\"").Append(side).Append("\",\"weights\":["); + for (int i = 0; i < FirebatNaxxFeatureExtractor.FeatureSchema.Count; i++) + { + if (i > 0) + sb.Append(','); + sb.Append(i == 0 ? biasWeight.ToString(System.Globalization.CultureInfo.InvariantCulture) : "0"); + } + sb.Append("]}"); + } + } + sb.Append("]}"); + return sb.ToString(); + } + + private static string BuildLinearPolicyModelJson() + { + var sb = new StringBuilder(); + sb.Append("{\"schema\":\"firebat_naxx_linear_value_v1\",\"featureSchema\":["); + for (int i = 0; i < FirebatNaxxFeatureExtractor.FeatureSchema.Count; i++) + { + if (i > 0) + sb.Append(','); + sb.Append('"').Append(FirebatNaxxFeatureExtractor.FeatureSchema[i]).Append('"'); + } + sb.Append("],\"actionFeatureSchema\":[\"bias\",\"type_end_turn\",\"type_play_card\"],"); + sb.Append("\"models\":[{\"kind\":\"legal_value\",\"side\":\"combo_druid\",\"weights\":["); + for (int i = 0; i < FirebatNaxxFeatureExtractor.FeatureSchema.Count; i++) + { + if (i > 0) + sb.Append(','); + sb.Append(i == 0 ? "0.1" : "0"); + } + sb.Append("]}],\"policyModels\":[{\"kind\":\"legal_policy\",\"side\":\"combo_druid\",\"weights\":[0,-4,4]}]}"); + return sb.ToString(); + } + } +} diff --git a/core-extensions/SabberStoneBasicAI/SabberStoneBasicAI.csproj b/core-extensions/SabberStoneBasicAI/SabberStoneBasicAI.csproj index a2babf8e5..80076e73f 100644 --- a/core-extensions/SabberStoneBasicAI/SabberStoneBasicAI.csproj +++ b/core-extensions/SabberStoneBasicAI/SabberStoneBasicAI.csproj @@ -1,7 +1,11 @@  - netstandard2.0 + netstandard2.0;net10.0 + + + + Exe diff --git a/core-extensions/SabberStoneBasicAI/src/Bots/AlternatingTurnSearch.cs b/core-extensions/SabberStoneBasicAI/src/Bots/AlternatingTurnSearch.cs new file mode 100644 index 000000000..4976a2956 --- /dev/null +++ b/core-extensions/SabberStoneBasicAI/src/Bots/AlternatingTurnSearch.cs @@ -0,0 +1,191 @@ +#region copyright +// SabberStone, Hearthstone Simulator in C# .NET Core +// Copyright (C) 2017-2019 SabberStone Team, darkfriend77 & rnilva +// +// SabberStone is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License. +// SabberStone is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +#endregion +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using SabberStoneCore.Enums; +using SabberStoneCore.Model; +using SabberStoneCore.Tasks.PlayerTasks; + +namespace SabberStoneBasicAI.Bots +{ + public sealed class AlternatingTurnSearch : ISearch + { + public string Name => "alternating-turn-search"; + + public BotTurnDecision Search(Game game, int playerId, IEvaluator evaluator, IBotPolicy opponentPolicy, SearchConfig config) + { + var watch = Stopwatch.StartNew(); + int nodesVisited = 0; + List candidates = TurnLineEnumerator.Enumerate( + game, + playerId, + evaluator, + config.MaxDepth, + config.MaxWidth, + config.CandidateLines, + watch, + config.TimeBudgetMs, + ref nodesVisited); + + if (candidates.Count == 0) + return Fallback(game, watch, nodesVisited); + + TurnLine best = null; + int bestScore = int.MinValue; + foreach (TurnLine candidate in candidates) + { + if (watch.ElapsedMilliseconds >= config.TimeBudgetMs) + break; + + int score = ScoreAfterOpponentReply(candidate.Game, playerId, evaluator, opponentPolicy, config.ForOpponent(), watch, config.TimeBudgetMs, ref nodesVisited); + if (best == null || score > bestScore) + { + best = candidate; + bestScore = score; + } + } + + if (best == null) + best = candidates.OrderByDescending(candidate => candidate.Score).First(); + + watch.Stop(); + return new BotTurnDecision(best.Tasks, bestScore, watch.Elapsed, Name, nodesVisited); + } + + private static int ScoreAfterOpponentReply(Game game, int playerId, IEvaluator evaluator, IBotPolicy opponentPolicy, + SearchConfig opponentConfig, Stopwatch watch, int totalTimeBudgetMs, ref int nodesVisited) + { + if (game.State != State.RUNNING || game.CurrentPlayer.Id == playerId || opponentPolicy == null) + return evaluator.Evaluate(game, playerId).Score; + + int remainingMs = Math.Max(50, totalTimeBudgetMs - (int)watch.ElapsedMilliseconds); + opponentConfig.TimeBudgetMs = Math.Min(opponentConfig.TimeBudgetMs, remainingMs); + List replies = TurnLineEnumerator.Enumerate( + game, + game.CurrentPlayer.Id, + opponentPolicy.Evaluator, + opponentConfig.MaxDepth, + opponentConfig.MaxWidth, + opponentConfig.CandidateLines, + watch, + totalTimeBudgetMs, + ref nodesVisited); + + if (replies.Count == 0) + return evaluator.Evaluate(game, playerId).Score; + + TurnLine opponentBest = replies.OrderByDescending(reply => reply.Score).First(); + return evaluator.Evaluate(opponentBest.Game, playerId).Score; + } + + private static BotTurnDecision Fallback(Game game, Stopwatch watch, int nodesVisited) + { + PlayerTask option = game.CurrentPlayer.Options().FirstOrDefault(task => !(task is ConcedeTask)); + watch.Stop(); + return new BotTurnDecision(option == null ? new List() : new List { option }, 0, watch.Elapsed, "fallback", nodesVisited); + } + } + + internal sealed class TurnLine + { + public TurnLine(Game game, IReadOnlyList tasks, int score) + { + Game = game; + Tasks = tasks; + Score = score; + } + + public Game Game { get; } + public IReadOnlyList Tasks { get; } + public int Score { get; } + } + + internal static class TurnLineEnumerator + { + public static List Enumerate(Game root, int playerId, IEvaluator evaluator, int maxDepth, int maxWidth, + int maxLines, Stopwatch watch, int totalTimeBudgetMs, ref int nodesVisited) + { + var frontier = new List + { + new TurnLine(root.Clone(resetRandomSeed: false), new List(), evaluator.Evaluate(root, playerId).Score) + }; + var leaves = new List(); + + for (int depth = 0; depth < maxDepth && frontier.Count > 0; depth++) + { + var nextByHash = new Dictionary(); + foreach (TurnLine line in frontier) + { + if (watch.ElapsedMilliseconds >= totalTimeBudgetMs) + break; + + Game lineGame = line.Game; + if (lineGame.State != State.RUNNING || lineGame.CurrentPlayer.Id != playerId) + { + leaves.Add(line); + continue; + } + + List options = lineGame.CurrentPlayer.Options() + .Where(task => !(task is ConcedeTask)) + .ToList(); + + if (options.Count == 0) + { + leaves.Add(line); + continue; + } + + foreach (PlayerTask option in options) + { + if (watch.ElapsedMilliseconds >= totalTimeBudgetMs) + break; + + Game clone = lineGame.Clone(resetRandomSeed: false); + if (!clone.Process(option)) + continue; + + nodesVisited++; + var tasks = new List(line.Tasks) { option }; + int score = evaluator.Evaluate(clone, playerId).Score; + var next = new TurnLine(clone, tasks, score); + + if (clone.State != State.RUNNING || clone.CurrentPlayer.Id != playerId || option is EndTurnTask) + { + leaves.Add(next); + continue; + } + + string hash = clone.Hash(GameTag.LAST_CARD_PLAYED, GameTag.ENTITY_ID); + if (!nextByHash.TryGetValue(hash, out TurnLine existing) || next.Score > existing.Score) + nextByHash[hash] = next; + } + } + + frontier = nextByHash.Values + .OrderByDescending(line => line.Score) + .Take(maxWidth) + .ToList(); + } + + leaves.AddRange(frontier); + return leaves + .OrderByDescending(line => line.Score) + .Take(maxLines) + .ToList(); + } + } +} diff --git a/core-extensions/SabberStoneBasicAI/src/Bots/BotInterfaces.cs b/core-extensions/SabberStoneBasicAI/src/Bots/BotInterfaces.cs new file mode 100644 index 000000000..31f23de21 --- /dev/null +++ b/core-extensions/SabberStoneBasicAI/src/Bots/BotInterfaces.cs @@ -0,0 +1,126 @@ +#region copyright +// SabberStone, Hearthstone Simulator in C# .NET Core +// Copyright (C) 2017-2019 SabberStone Team, darkfriend77 & rnilva +// +// SabberStone is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License. +// SabberStone is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +#endregion +using System; +using System.Collections.Generic; +using SabberStoneCore.Model; +using SabberStoneCore.Model.Entities; +using SabberStoneCore.Tasks.PlayerTasks; + +namespace SabberStoneBasicAI.Bots +{ + public interface IBotPolicy + { + string Name { get; } + IEvaluator Evaluator { get; } + List Mulligan(Game game, Controller controller); + BotTurnDecision SelectTurn(Game game, Controller controller, BotTurnContext context); + } + + public interface IEvaluator + { + string Name { get; } + EvaluationResult Evaluate(Game game, int playerId); + } + + public interface ISearch + { + string Name { get; } + BotTurnDecision Search(Game game, int playerId, IEvaluator evaluator, IBotPolicy opponentPolicy, SearchConfig config); + } + + public sealed class BotTurnContext + { + public BotTurnContext(SearchConfig searchConfig, IBotPolicy opponentPolicy) + { + SearchConfig = searchConfig; + OpponentPolicy = opponentPolicy; + } + + public SearchConfig SearchConfig { get; } + public IBotPolicy OpponentPolicy { get; } + } + + public sealed class BotTurnDecision + { + public BotTurnDecision(IReadOnlyList tasks, int score, TimeSpan elapsed, string searchName, int nodesVisited, + IReadOnlyDictionary rootActionTargets = null) + { + Tasks = tasks; + Score = score; + Elapsed = elapsed; + SearchName = searchName; + NodesVisited = nodesVisited; + RootActionTargets = rootActionTargets == null + ? new Dictionary() + : new Dictionary(rootActionTargets); + } + + public IReadOnlyList Tasks { get; } + public int Score { get; } + public TimeSpan Elapsed { get; } + public string SearchName { get; } + public int NodesVisited { get; } + public IReadOnlyDictionary RootActionTargets { get; } + } + + public sealed class EvaluationResult + { + public EvaluationResult(int score, IDictionary features = null) + { + Score = score; + Features = features == null + ? new Dictionary() + : new Dictionary(features); + } + + public int Score { get; } + public IReadOnlyDictionary Features { get; } + } + + public sealed class SearchConfig + { + public int MaxDepth { get; set; } = 8; + public int MaxWidth { get; set; } = 80; + public int CandidateLines { get; set; } = 12; + public int OpponentResponseLines { get; set; } = 8; + public int OpponentDepth { get; set; } = 6; + public int TimeBudgetMs { get; set; } = 2000; + public int MctsIterations { get; set; } = 160; + public double MctsExploration { get; set; } = 1.5; + + public SearchConfig Clone() + { + return new SearchConfig + { + MaxDepth = MaxDepth, + MaxWidth = MaxWidth, + CandidateLines = CandidateLines, + OpponentResponseLines = OpponentResponseLines, + OpponentDepth = OpponentDepth, + TimeBudgetMs = TimeBudgetMs, + MctsIterations = MctsIterations, + MctsExploration = MctsExploration + }; + } + + public SearchConfig ForOpponent() + { + var clone = Clone(); + clone.MaxDepth = OpponentDepth; + clone.CandidateLines = OpponentResponseLines; + clone.TimeBudgetMs = Math.Max(50, TimeBudgetMs / 3); + return clone; + } + } +} diff --git a/core-extensions/SabberStoneBasicAI/src/Bots/FirebatNaxxBenchmarkRunner.cs b/core-extensions/SabberStoneBasicAI/src/Bots/FirebatNaxxBenchmarkRunner.cs new file mode 100644 index 000000000..10a5eebe9 --- /dev/null +++ b/core-extensions/SabberStoneBasicAI/src/Bots/FirebatNaxxBenchmarkRunner.cs @@ -0,0 +1,324 @@ +#region copyright +// SabberStone, Hearthstone Simulator in C# .NET Core +// Copyright (C) 2017-2019 SabberStone Team, darkfriend77 & rnilva +// +// SabberStone is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License. +// SabberStone is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +#endregion +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using SabberStoneCore.Enums; +using SabberStoneCore.Model; +using SabberStoneCore.Model.Entities; +using SabberStoneCore.Tasks.PlayerTasks; + +namespace SabberStoneBasicAI.Bots +{ + public sealed class FirebatNaxxGameResult + { + public int Seed { get; set; } + public string Player1Policy { get; set; } + public string Player2Policy { get; set; } + public PlayState Player1Result { get; set; } + public PlayState Player2Result { get; set; } + public int Turns { get; set; } + public int Decisions { get; set; } + public int NodesVisited { get; set; } + public TimeSpan SearchElapsed { get; set; } + public string Error { get; set; } + public bool IsComplete => Error == null && Player1Result != PlayState.INVALID && Player2Result != PlayState.INVALID; + } + + public sealed class FirebatNaxxScenarioResult + { + private readonly List _games = new List(); + + public FirebatNaxxScenarioResult(string name, string player1Policy, string player2Policy) + { + Name = name; + Player1Policy = player1Policy; + Player2Policy = player2Policy; + } + + public string Name { get; } + public string Player1Policy { get; } + public string Player2Policy { get; } + public IReadOnlyList Games => _games; + public int Player1Wins => _games.Count(game => game.Player1Result == PlayState.WON); + public int Player2Wins => _games.Count(game => game.Player2Result == PlayState.WON); + public int Ties => _games.Count(game => game.Player1Result == PlayState.TIED || game.Player2Result == PlayState.TIED); + public int Unfinished => _games.Count(game => game.Error == null && game.Player1Result != PlayState.WON && game.Player2Result != PlayState.WON && game.Player1Result != PlayState.TIED && game.Player2Result != PlayState.TIED); + public int Errors => _games.Count(game => game.Error != null); + public double AverageTurns => _games.Count == 0 ? 0 : _games.Average(game => game.Turns); + public double AverageMoveMs => _games.Sum(game => game.Decisions) == 0 ? 0 : _games.Sum(game => game.SearchElapsed.TotalMilliseconds) / _games.Sum(game => game.Decisions); + public int NodesVisited => _games.Sum(game => game.NodesVisited); + + public void Add(FirebatNaxxGameResult result) + { + _games.Add(result); + } + } + + public sealed class FirebatNaxxBenchmarkReport + { + public FirebatNaxxBenchmarkReport(IReadOnlyList scenarios) + { + Scenarios = scenarios; + } + + public IReadOnlyList Scenarios { get; } + } + + public static class FirebatNaxxBenchmarkRunner + { + public static FirebatNaxxBenchmarkReport RunBenchmark(int games, long seed, SearchConfig config, int maxTurns = 80, bool verbose = false) + { + var scenarios = new List + { + RunScenario("baseline mirror", FirebatNaxxPolicies.BaselineDruid(), FirebatNaxxPolicies.BaselineZoo(), games, seed, config, maxTurns, verbose), + RunScenario("candidate druid vs baseline zoo", FirebatNaxxPolicies.CandidateDruid(), FirebatNaxxPolicies.BaselineZoo(), games, seed + 1000, config, maxTurns, verbose), + RunScenario("baseline druid vs candidate zoo", FirebatNaxxPolicies.BaselineDruid(), FirebatNaxxPolicies.CandidateZoo(), games, seed + 2000, config, maxTurns, verbose), + RunScenario("candidate mirror", FirebatNaxxPolicies.CandidateDruid(), FirebatNaxxPolicies.CandidateZoo(), games, seed + 3000, config, maxTurns, verbose) + }; + + return new FirebatNaxxBenchmarkReport(scenarios); + } + + public static FirebatNaxxBenchmarkReport RunPolicyValueBenchmark(int games, long seed, SearchConfig config, int maxTurns = 80, bool verbose = false) + { + var scenarios = new List + { + RunScenario("baseline mirror", FirebatNaxxPolicies.BaselineDruid(), FirebatNaxxPolicies.BaselineZoo(), games, seed, config, maxTurns, verbose), + RunScenario("policy-value druid vs baseline zoo", FirebatNaxxPolicies.PolicyValueDruid(), FirebatNaxxPolicies.BaselineZoo(), games, seed + 1000, config, maxTurns, verbose), + RunScenario("baseline druid vs policy-value zoo", FirebatNaxxPolicies.BaselineDruid(), FirebatNaxxPolicies.PolicyValueZoo(), games, seed + 2000, config, maxTurns, verbose), + RunScenario("policy-value mirror", FirebatNaxxPolicies.PolicyValueDruid(), FirebatNaxxPolicies.PolicyValueZoo(), games, seed + 3000, config, maxTurns, verbose) + }; + + return new FirebatNaxxBenchmarkReport(scenarios); + } + + public static FirebatNaxxBenchmarkReport RunTrainedPolicyValueBenchmark(string modelPath, int games, long seed, SearchConfig config, int maxTurns = 80, bool verbose = false) + { + var scenarios = new List + { + RunScenario("baseline mirror", FirebatNaxxPolicies.BaselineDruid(), FirebatNaxxPolicies.BaselineZoo(), games, seed, config, maxTurns, verbose), + RunScenario("trained policy-value druid vs baseline zoo", FirebatNaxxPolicies.TrainedPolicyValueDruid(modelPath), FirebatNaxxPolicies.BaselineZoo(), games, seed + 1000, config, maxTurns, verbose), + RunScenario("baseline druid vs trained policy-value zoo", FirebatNaxxPolicies.BaselineDruid(), FirebatNaxxPolicies.TrainedPolicyValueZoo(modelPath), games, seed + 2000, config, maxTurns, verbose), + RunScenario("trained policy-value mirror", FirebatNaxxPolicies.TrainedPolicyValueDruid(modelPath), FirebatNaxxPolicies.TrainedPolicyValueZoo(modelPath), games, seed + 3000, config, maxTurns, verbose) + }; + + return new FirebatNaxxBenchmarkReport(scenarios); + } + + public static FirebatNaxxBenchmarkReport RunTrainedPolicyValueArena(string candidateModelPath, string incumbentModelPath, int games, long seed, SearchConfig config, int maxTurns = 80, bool verbose = false) + { + var scenarios = new List + { + RunScenario("candidate druid vs incumbent zoo", FirebatNaxxPolicies.TrainedPolicyValueDruid(candidateModelPath), FirebatNaxxPolicies.TrainedPolicyValueZoo(incumbentModelPath), games, seed, config, maxTurns, verbose), + RunScenario("incumbent druid vs candidate zoo", FirebatNaxxPolicies.TrainedPolicyValueDruid(incumbentModelPath), FirebatNaxxPolicies.TrainedPolicyValueZoo(candidateModelPath), games, seed + 1000, config, maxTurns, verbose), + RunScenario("candidate mirror", FirebatNaxxPolicies.TrainedPolicyValueDruid(candidateModelPath), FirebatNaxxPolicies.TrainedPolicyValueZoo(candidateModelPath), games, seed + 2000, config, maxTurns, verbose), + RunScenario("incumbent mirror", FirebatNaxxPolicies.TrainedPolicyValueDruid(incumbentModelPath), FirebatNaxxPolicies.TrainedPolicyValueZoo(incumbentModelPath), games, seed + 3000, config, maxTurns, verbose) + }; + + return new FirebatNaxxBenchmarkReport(scenarios); + } + + public static int CandidateArenaWins(FirebatNaxxBenchmarkReport report) + { + FirebatNaxxScenarioResult candidateDruid = report.Scenarios.FirstOrDefault(s => s.Name == "candidate druid vs incumbent zoo"); + FirebatNaxxScenarioResult candidateZoo = report.Scenarios.FirstOrDefault(s => s.Name == "incumbent druid vs candidate zoo"); + return (candidateDruid?.Player1Wins ?? 0) + (candidateZoo?.Player2Wins ?? 0); + } + + public static int CandidateArenaGames(FirebatNaxxBenchmarkReport report) + { + FirebatNaxxScenarioResult candidateDruid = report.Scenarios.FirstOrDefault(s => s.Name == "candidate druid vs incumbent zoo"); + FirebatNaxxScenarioResult candidateZoo = report.Scenarios.FirstOrDefault(s => s.Name == "incumbent druid vs candidate zoo"); + return (candidateDruid?.Games.Count ?? 0) + (candidateZoo?.Games.Count ?? 0); + } + + public static FirebatNaxxScenarioResult RunCandidateMirror(int games, long seed, SearchConfig config, int maxTurns = 80, bool verbose = false) + { + return RunScenario("candidate mirror", FirebatNaxxPolicies.CandidateDruid(), FirebatNaxxPolicies.CandidateZoo(), games, seed, config, maxTurns, verbose); + } + + public static FirebatNaxxScenarioResult RunPolicyValueMirror(int games, long seed, SearchConfig config, int maxTurns = 80, bool verbose = false) + { + return RunScenario("policy-value mirror", FirebatNaxxPolicies.PolicyValueDruid(), FirebatNaxxPolicies.PolicyValueZoo(), games, seed, config, maxTurns, verbose); + } + + public static FirebatNaxxScenarioResult RunTrainedPolicyValueMirror(string modelPath, int games, long seed, SearchConfig config, int maxTurns = 80, bool verbose = false) + { + return RunScenario("trained policy-value mirror", FirebatNaxxPolicies.TrainedPolicyValueDruid(modelPath), FirebatNaxxPolicies.TrainedPolicyValueZoo(modelPath), games, seed, config, maxTurns, verbose); + } + + public static void PrintReport(FirebatNaxxBenchmarkReport report) + { + foreach (FirebatNaxxScenarioResult scenario in report.Scenarios) + PrintScenario(scenario); + + FirebatNaxxScenarioResult candidateDruid = report.Scenarios.FirstOrDefault(s => s.Name == "candidate druid vs baseline zoo") + ?? report.Scenarios.FirstOrDefault(s => s.Name == "policy-value druid vs baseline zoo") + ?? report.Scenarios.FirstOrDefault(s => s.Name == "trained policy-value druid vs baseline zoo"); + FirebatNaxxScenarioResult candidateZoo = report.Scenarios.FirstOrDefault(s => s.Name == "baseline druid vs candidate zoo") + ?? report.Scenarios.FirstOrDefault(s => s.Name == "baseline druid vs policy-value zoo") + ?? report.Scenarios.FirstOrDefault(s => s.Name == "baseline druid vs trained policy-value zoo"); + if (candidateDruid != null && candidateZoo != null) + { + int candidateWins = candidateDruid.Player1Wins + candidateZoo.Player2Wins; + int games = candidateDruid.Games.Count + candidateZoo.Games.Count; + Console.WriteLine(); + Console.WriteLine($"Candidate vs baseline aggregate: {candidateWins}/{games} wins ({Percent(candidateWins, games):0.0}%)."); + } + } + + public static void RunTuning(int games, long seed, SearchConfig baseConfig, int maxTurns = 80) + { + var presets = new Dictionary + { + ["fast"] = new SearchConfig { MaxDepth = 6, MaxWidth = 50, CandidateLines = 8, OpponentDepth = 4, OpponentResponseLines = 5, TimeBudgetMs = Math.Min(1000, baseConfig.TimeBudgetMs) }, + ["balanced"] = baseConfig.Clone(), + ["wide"] = new SearchConfig { MaxDepth = baseConfig.MaxDepth, MaxWidth = Math.Max(baseConfig.MaxWidth, 140), CandidateLines = 18, OpponentDepth = baseConfig.OpponentDepth, OpponentResponseLines = 10, TimeBudgetMs = baseConfig.TimeBudgetMs }, + ["deep"] = new SearchConfig { MaxDepth = Math.Max(baseConfig.MaxDepth, 10), MaxWidth = baseConfig.MaxWidth, CandidateLines = baseConfig.CandidateLines, OpponentDepth = Math.Max(baseConfig.OpponentDepth, 8), OpponentResponseLines = baseConfig.OpponentResponseLines, TimeBudgetMs = baseConfig.TimeBudgetMs } + }; + + string bestName = null; + int bestWins = int.MinValue; + SearchConfig bestConfig = null; + foreach (KeyValuePair preset in presets) + { + FirebatNaxxBenchmarkReport report = RunBenchmark(games, seed, preset.Value, maxTurns); + FirebatNaxxScenarioResult candidateDruid = report.Scenarios.First(s => s.Name == "candidate druid vs baseline zoo"); + FirebatNaxxScenarioResult candidateZoo = report.Scenarios.First(s => s.Name == "baseline druid vs candidate zoo"); + int wins = candidateDruid.Player1Wins + candidateZoo.Player2Wins; + Console.WriteLine($"{preset.Key}: candidate aggregate {wins}/{games * 2} ({Percent(wins, games * 2):0.0}%), avg move {report.Scenarios.Average(s => s.AverageMoveMs):0.0} ms"); + if (wins > bestWins) + { + bestWins = wins; + bestName = preset.Key; + bestConfig = preset.Value; + } + } + + Console.WriteLine(); + Console.WriteLine($"Best training preset: {bestName}"); + Console.WriteLine("Holdout:"); + PrintReport(RunBenchmark(games, seed + 100000, bestConfig, maxTurns)); + } + + private static FirebatNaxxScenarioResult RunScenario(string name, IBotPolicy player1Policy, IBotPolicy player2Policy, + int games, long seed, SearchConfig config, int maxTurns, bool verbose) + { + var result = new FirebatNaxxScenarioResult(name, player1Policy.Name, player2Policy.Name); + for (int i = 0; i < games; i++) + result.Add(RunGame(player1Policy, player2Policy, seed + i, config, maxTurns, verbose)); + return result; + } + + private static FirebatNaxxGameResult RunGame(IBotPolicy player1Policy, IBotPolicy player2Policy, long seed, + SearchConfig config, int maxTurns, bool verbose) + { + var result = new FirebatNaxxGameResult + { + Seed = (int)seed, + Player1Policy = player1Policy.Name, + Player2Policy = player2Policy.Name, + Player1Result = PlayState.INVALID, + Player2Result = PlayState.INVALID + }; + + try + { + Game game = FirebatNaxxGameFactory.Create(seed); + game.StartGame(); + ApplyMulligan(game, game.Player1, player1Policy); + ApplyMulligan(game, game.Player2, player2Policy); + game.MainReady(); + + int decisions = 0; + int nodes = 0; + TimeSpan searchElapsed = TimeSpan.Zero; + while (game.State != State.COMPLETE && game.Turn <= maxTurns && decisions < 1000) + { + IBotPolicy currentPolicy = game.CurrentPlayer == game.Player1 ? player1Policy : player2Policy; + IBotPolicy opponentPolicy = game.CurrentPlayer == game.Player1 ? player2Policy : player1Policy; + int currentPlayerId = game.CurrentPlayer.Id; + + BotTurnDecision decision = currentPolicy.SelectTurn(game, game.CurrentPlayer, new BotTurnContext(config, opponentPolicy)); + decisions++; + nodes += decision.NodesVisited; + searchElapsed += decision.Elapsed; + + if (!ProcessDecision(game, currentPlayerId, decision, verbose)) + break; + } + + result.Player1Result = game.Player1.PlayState; + result.Player2Result = game.Player2.PlayState; + result.Turns = game.Turn; + result.Decisions = decisions; + result.NodesVisited = nodes; + result.SearchElapsed = searchElapsed; + } + catch (Exception ex) + { + result.Error = ex.GetType().Name + ": " + ex.Message; + } + + return result; + } + + private static void ApplyMulligan(Game game, Controller controller, IBotPolicy policy) + { + if (controller.Choice == null) + return; + + game.Process(ChooseTask.Mulligan(controller, policy.Mulligan(game, controller))); + } + + private static bool ProcessDecision(Game game, int playerId, BotTurnDecision decision, bool verbose) + { + IReadOnlyList tasks = decision.Tasks; + if (tasks.Count == 0) + tasks = game.CurrentPlayer.Options().Where(task => !(task is ConcedeTask)).Take(1).ToList(); + + foreach (PlayerTask task in tasks) + { + if (game.State != State.RUNNING || game.CurrentPlayer.Id != playerId) + break; + + if (verbose) + Console.WriteLine($"{decision.SearchName} score={decision.Score} {task.FullPrint()}"); + + if (!game.Process(task)) + return false; + } + + return true; + } + + private static void PrintScenario(FirebatNaxxScenarioResult scenario) + { + Console.WriteLine(); + Console.WriteLine(scenario.Name); + Console.WriteLine($"{FirebatNaxxGameFactory.DruidName} ({scenario.Player1Policy}): {scenario.Player1Wins}/{scenario.Games.Count} wins ({Percent(scenario.Player1Wins, scenario.Games.Count):0.0}%)"); + Console.WriteLine($"{FirebatNaxxGameFactory.WarlockName} ({scenario.Player2Policy}): {scenario.Player2Wins}/{scenario.Games.Count} wins ({Percent(scenario.Player2Wins, scenario.Games.Count):0.0}%)"); + Console.WriteLine($"ties={scenario.Ties}, unfinished={scenario.Unfinished}, errors={scenario.Errors}, avgTurns={scenario.AverageTurns:0.0}, avgMoveMs={scenario.AverageMoveMs:0.0}, nodes={scenario.NodesVisited}"); + foreach (FirebatNaxxGameResult game in scenario.Games.Where(game => game.Error != null).Take(3)) + Console.WriteLine($" seed {game.Seed}: {game.Error}"); + } + + private static double Percent(int count, int total) + { + return total == 0 ? 0 : count * 100.0 / total; + } + } +} diff --git a/core-extensions/SabberStoneBasicAI/src/Bots/FirebatNaxxEvaluators.cs b/core-extensions/SabberStoneBasicAI/src/Bots/FirebatNaxxEvaluators.cs new file mode 100644 index 000000000..6c7823b87 --- /dev/null +++ b/core-extensions/SabberStoneBasicAI/src/Bots/FirebatNaxxEvaluators.cs @@ -0,0 +1,216 @@ +#region copyright +// SabberStone, Hearthstone Simulator in C# .NET Core +// Copyright (C) 2017-2019 SabberStone Team, darkfriend77 & rnilva +// +// SabberStone is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License. +// SabberStone is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +#endregion +using System; +using System.Collections.Generic; +using System.Linq; +using SabberStoneBasicAI.Score; +using SabberStoneCore.Enums; +using SabberStoneCore.Model; +using SabberStoneCore.Model.Entities; + +namespace SabberStoneBasicAI.Bots +{ + public sealed class ScoreEvaluatorAdapter : IEvaluator + { + private readonly IScore _score; + + public ScoreEvaluatorAdapter(string name, IScore score) + { + Name = name; + _score = score; + } + + public string Name { get; } + + public EvaluationResult Evaluate(Game game, int playerId) + { + _score.Controller = game.ControllerById(playerId); + return new EvaluationResult(_score.Rate()); + } + } + + public sealed class FirebatNaxxDruidEvaluator : IEvaluator + { + public string Name => "Firebat Naxx Combo Druid evaluator"; + + public EvaluationResult Evaluate(Game game, int playerId) + { + Controller c = game.ControllerById(playerId); + Controller op = c.Opponent; + + if (op.PlayState == PlayState.LOST || op.PlayState == PlayState.CONCEDED || op.Hero.Health < 1) + return new EvaluationResult(int.MaxValue); + if (c.PlayState == PlayState.LOST || c.PlayState == PlayState.CONCEDED || c.Hero.Health < 1) + return new EvaluationResult(int.MinValue); + + int boardAttack = BoardAttack(c); + int opponentAttack = BoardAttack(op); + int chargeComboDamage = ComboDamageFromHand(c); + int publicLethalReach = boardAttack + chargeComboDamage; + int tauntWall = op.BoardZone.Where(m => m.HasTaunt).Sum(m => m.Health + m.AttackDamage); + + var features = new Dictionary + { + ["health"] = (c.Hero.Health - op.Hero.Health) * 30, + ["armor"] = (c.Hero.Armor - op.Hero.Armor) * 18, + ["mana"] = c.BaseMana * 80 + c.RemainingMana * 8, + ["board_attack"] = (boardAttack - opponentAttack) * 55, + ["board_health"] = (BoardHealth(c) - BoardHealth(op)) * 22, + ["enemy_board_pressure"] = -opponentAttack * 110 - op.BoardZone.Count * 70, + ["taunt_wall"] = -tauntWall * 190, + ["hand"] = c.HandZone.Count * 45, + ["draw"] = CountInHand(c, "Ancient of Lore") * 150 + CountInHand(c, "Azure Drake") * 80, + ["ramp"] = CountInHand(c, "Wild Growth") * (c.BaseMana < 7 ? 180 : 15), + ["combo_pieces"] = ComboPieceScore(c), + ["lethal_reach"] = Math.Max(0, publicLethalReach - EffectiveHeroHealth(op)) * 1800, + ["survival_floor"] = c.Hero.Health <= opponentAttack + 5 ? -2200 : 0 + }; + + return new EvaluationResult(features.Values.Sum(), features); + } + + private static int ComboDamageFromHand(Controller c) + { + int force = CountInHand(c, "Force of Nature"); + int roars = CountInHand(c, "Savage Roar"); + if (force == 0 || roars == 0) + return 0; + + int damage = 6 + roars * 8; + if (c.RemainingMana >= 9 && roars > 1) + damage += 8; + return damage; + } + + private static int ComboPieceScore(Controller c) + { + int force = CountInHand(c, "Force of Nature"); + int roars = CountInHand(c, "Savage Roar"); + int innervates = CountInHand(c, "Innervate"); + int score = force * 650 + roars * 450 + innervates * 110; + if (force > 0 && roars > 0) + score += c.BaseMana >= 9 || (c.BaseMana >= 7 && innervates > 0) ? 1800 : 850; + return score; + } + + private static int BoardAttack(Controller c) + { + return c.BoardZone.Sum(m => Math.Max(0, m.AttackDamage)) + c.Hero.TotalAttackDamage; + } + + private static int BoardHealth(Controller c) + { + return c.BoardZone.Sum(m => Math.Max(0, m.Health)); + } + + private static int EffectiveHeroHealth(Controller c) + { + return c.Hero.Health + c.Hero.Armor; + } + + private static int CountInHand(Controller c, string name) + { + return c.HandZone.Count(card => card.Card.Name == name); + } + } + + public sealed class FirebatNaxxZooEvaluator : IEvaluator + { + public string Name => "Firebat Naxx Zoo Warlock evaluator"; + + public EvaluationResult Evaluate(Game game, int playerId) + { + Controller c = game.ControllerById(playerId); + Controller op = c.Opponent; + + if (op.PlayState == PlayState.LOST || op.PlayState == PlayState.CONCEDED || op.Hero.Health < 1) + return new EvaluationResult(int.MaxValue); + if (c.PlayState == PlayState.LOST || c.PlayState == PlayState.CONCEDED || c.Hero.Health < 1) + return new EvaluationResult(int.MinValue); + + int boardAttack = BoardAttack(c); + int opponentAttack = BoardAttack(op); + int burnReach = BurnFromHand(c); + int tauntWall = op.BoardZone.Where(m => m.HasTaunt).Sum(m => m.Health + m.AttackDamage); + + var features = new Dictionary + { + ["face_damage"] = (30 - EffectiveHeroHealth(op)) * 115, + ["self_health"] = (c.Hero.Health - 10) * 18, + ["board_count"] = c.BoardZone.Count * 185 - op.BoardZone.Count * 85, + ["board_attack"] = (boardAttack - opponentAttack) * 95, + ["sticky_board"] = StickyBoardScore(c), + ["undertaker"] = c.BoardZone.Where(m => m.Card.Name == "Undertaker").Sum(m => m.AttackDamage + m.Health) * 170, + ["taunt_wall"] = -tauntWall * 260, + ["hand"] = c.HandZone.Count * 18, + ["curve"] = CurvePressure(c), + ["lethal_reach"] = Math.Max(0, boardAttack + burnReach - EffectiveHeroHealth(op)) * 2200, + ["druid_sweep_risk"] = SweepRisk(c) * -70 + }; + + return new EvaluationResult(features.Values.Sum(), features); + } + + private static int BurnFromHand(Controller c) + { + return CountInHand(c, "Soulfire") * 4 + + CountInHand(c, "Power Overwhelming") * 4 + + CountInHand(c, "Doomguard") * 5 + + CountInHand(c, "Abusive Sergeant") * 2 + + CountInHand(c, "Dark Iron Dwarf") * 2; + } + + private static int StickyBoardScore(Controller c) + { + int score = 0; + foreach (Minion minion in c.BoardZone) + { + if (minion.Card.Name == "Haunted Creeper" || minion.Card.Name == "Nerubian Egg" || minion.Card.Name == "Harvest Golem") + score += 170; + score += Math.Max(0, minion.Health - 1) * 20; + } + return score; + } + + private static int CurvePressure(Controller c) + { + if (c.RemainingMana < 1) + return 0; + + return c.HandZone + .Where(card => card.Cost <= c.RemainingMana) + .Sum(card => Math.Max(0, 4 - card.Cost) * 35); + } + + private static int SweepRisk(Controller c) + { + return c.BoardZone.Count(m => m.Health <= 1) * 2 + c.BoardZone.Count(m => m.Health <= 4); + } + + private static int BoardAttack(Controller c) + { + return c.BoardZone.Sum(m => Math.Max(0, m.AttackDamage)) + c.Hero.TotalAttackDamage; + } + + private static int EffectiveHeroHealth(Controller c) + { + return c.Hero.Health + c.Hero.Armor; + } + + private static int CountInHand(Controller c, string name) + { + return c.HandZone.Count(card => card.Card.Name == name); + } + } +} diff --git a/core-extensions/SabberStoneBasicAI/src/Bots/FirebatNaxxGameFactory.cs b/core-extensions/SabberStoneBasicAI/src/Bots/FirebatNaxxGameFactory.cs new file mode 100644 index 000000000..d473b74c2 --- /dev/null +++ b/core-extensions/SabberStoneBasicAI/src/Bots/FirebatNaxxGameFactory.cs @@ -0,0 +1,65 @@ +#region copyright +// SabberStone, Hearthstone Simulator in C# .NET Core +// Copyright (C) 2017-2019 SabberStone Team, darkfriend77 & rnilva +// +// SabberStone is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License. +// SabberStone is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +#endregion +using System; +using System.Collections.Generic; +using System.Linq; +using SabberStoneBasicAI.Meta; +using SabberStoneCore.Config; +using SabberStoneCore.Enums; +using SabberStoneCore.Model; + +namespace SabberStoneBasicAI.Bots +{ + public static class FirebatNaxxGameFactory + { + public const string DruidName = "Firebat Combo Druid"; + public const string WarlockName = "Firebat Zoo Warlock"; + + public static Game Create(long? seed) + { + ValidateDecks(); + return new Game(new GameConfig + { + StartPlayer = -1, + Player1Name = DruidName, + Player1HeroClass = CardClass.DRUID, + Player1Deck = Decks.FirebatNaxxComboDruid, + Player2Name = WarlockName, + Player2HeroClass = CardClass.WARLOCK, + Player2Deck = Decks.FirebatNaxxZooWarlock, + FormatType = FormatType.FT_WILD, + FillDecks = false, + Shuffle = true, + SkipMulligan = false, + History = false, + Logging = false, + RandomSeed = seed + }); + } + + public static void ValidateDecks() + { + ValidateDeck("Combo Druid", Decks.FirebatNaxxComboDruid); + ValidateDeck("Zoo Warlock", Decks.FirebatNaxxZooWarlock); + } + + private static void ValidateDeck(string name, IReadOnlyCollection deck) + { + if (deck.Count != 30) + throw new InvalidOperationException($"{name} has {deck.Count} cards; expected 30."); + if (deck.Any(card => card == null)) + throw new InvalidOperationException($"{name} contains an unknown card id."); + } + } +} diff --git a/core-extensions/SabberStoneBasicAI/src/Bots/FirebatNaxxPolicies.cs b/core-extensions/SabberStoneBasicAI/src/Bots/FirebatNaxxPolicies.cs new file mode 100644 index 000000000..96dbad65a --- /dev/null +++ b/core-extensions/SabberStoneBasicAI/src/Bots/FirebatNaxxPolicies.cs @@ -0,0 +1,406 @@ +#region copyright +// SabberStone, Hearthstone Simulator in C# .NET Core +// Copyright (C) 2017-2019 SabberStone Team, darkfriend77 & rnilva +// +// SabberStone is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License. +// SabberStone is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +#endregion +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using SabberStoneBasicAI.Nodes; +using SabberStoneBasicAI.Score; +using SabberStoneCore.Model; +using SabberStoneCore.Model.Entities; +using SabberStoneCore.Tasks.PlayerTasks; + +namespace SabberStoneBasicAI.Bots +{ + public sealed class BaselinePolicy : IBotPolicy + { + private readonly IScore _score; + + public BaselinePolicy(string name, IScore score) + { + Name = name; + _score = score; + Evaluator = new ScoreEvaluatorAdapter($"{name} evaluator", score); + } + + public string Name { get; } + public IEvaluator Evaluator { get; } + + public List Mulligan(Game game, Controller controller) + { + if (controller.Choice == null) + return new List(); + + return _score.MulliganRule().Invoke(controller.Choice.Choices.Select(id => game.IdEntityDic[id]).ToList()); + } + + public BotTurnDecision SelectTurn(Game game, Controller controller, BotTurnContext context) + { + var watch = Stopwatch.StartNew(); + List solutions = OptionNode.GetSolutions( + game, + controller.Id, + _score, + context.SearchConfig.MaxDepth, + context.SearchConfig.MaxWidth); + + var tasks = new List(); + int score = 0; + if (solutions.Count > 0) + { + OptionNode best = solutions.OrderByDescending(solution => solution.Score).First(); + best.PlayerTasks(ref tasks); + score = best.Score; + } + + if (tasks.Count == 0) + { + PlayerTask fallback = controller.Options().FirstOrDefault(task => !(task is ConcedeTask)); + if (fallback != null) + tasks.Add(fallback); + } + + watch.Stop(); + return new BotTurnDecision(tasks, score, watch.Elapsed, "baseline-option-node", solutions.Count); + } + } + + public sealed class ConstantEvaluator : IEvaluator + { + public ConstantEvaluator(string name, int score = 0) + { + Name = name; + _score = score; + } + + private readonly int _score; + public string Name { get; } + + public EvaluationResult Evaluate(Game game, int playerId) + { + return new EvaluationResult(_score); + } + } + + public sealed class RandomActionPolicy : IBotPolicy + { + private readonly Random _random; + private readonly int _maxActionsPerTurn; + + public RandomActionPolicy(string name, int seed, int maxActionsPerTurn = 80) + { + Name = name; + _random = new Random(seed); + _maxActionsPerTurn = Math.Max(1, maxActionsPerTurn); + Evaluator = new ConstantEvaluator($"{name} evaluator"); + } + + public string Name { get; } + public IEvaluator Evaluator { get; } + + public List Mulligan(Game game, Controller controller) + { + if (controller.Choice == null) + return new List(); + + return controller.Choice.Choices + .Where(_ => _random.Next(2) == 0) + .ToList(); + } + + public BotTurnDecision SelectTurn(Game game, Controller controller, BotTurnContext context) + { + var watch = Stopwatch.StartNew(); + var tasks = new List(); + Game simulation = game.Clone(resetRandomSeed: false); + int actions = 0; + + while (simulation.State == SabberStoneCore.Enums.State.RUNNING && + simulation.CurrentPlayer.Id == controller.Id && + actions < _maxActionsPerTurn) + { + List options = simulation.CurrentPlayer.Options() + .Where(task => !(task is ConcedeTask)) + .ToList(); + List playable = options + .Where(task => !(task is EndTurnTask)) + .ToList(); + + if (playable.Count == 0) + { + PlayerTask endTurn = options.FirstOrDefault(task => task is EndTurnTask); + if (endTurn != null) + tasks.Add(endTurn); + break; + } + + PlayerTask selected = playable[_random.Next(playable.Count)]; + tasks.Add(selected); + if (!simulation.Process(selected)) + break; + actions++; + } + + if (simulation.State == SabberStoneCore.Enums.State.RUNNING && + simulation.CurrentPlayer.Id == controller.Id && + tasks.All(task => !(task is EndTurnTask))) + { + List options = simulation.CurrentPlayer.Options() + .Where(task => !(task is ConcedeTask)) + .ToList(); + bool hasPlayableAction = options.Any(task => !(task is EndTurnTask)); + PlayerTask endTurn = options.FirstOrDefault(task => task is EndTurnTask); + if (!hasPlayableAction && endTurn != null) + tasks.Add(endTurn); + } + + watch.Stop(); + return new BotTurnDecision(tasks, 0, watch.Elapsed, "random-actions-until-end-turn", tasks.Count); + } + } + + public sealed class SearchPolicy : IBotPolicy + { + private readonly Func> _mulligan; + private readonly ISearch _search; + + public SearchPolicy(string name, IEvaluator evaluator, ISearch search, Func> mulligan) + { + Name = name; + Evaluator = evaluator; + _search = search; + _mulligan = mulligan; + } + + public string Name { get; } + public IEvaluator Evaluator { get; } + + public List Mulligan(Game game, Controller controller) + { + return _mulligan(game, controller); + } + + public BotTurnDecision SelectTurn(Game game, Controller controller, BotTurnContext context) + { + return _search.Search(game, controller.Id, Evaluator, context.OpponentPolicy, context.SearchConfig); + } + } + + public sealed class PolicyValuePolicy : IPolicyValueBotPolicy + { + private readonly Func> _mulligan; + private readonly ISearch _search; + + public PolicyValuePolicy(string name, IPolicyValueNet policyValueNet, Func> mulligan, ISearch search = null) + { + Name = name; + PolicyValueNet = policyValueNet; + Evaluator = new PolicyValueEvaluator(policyValueNet); + _search = search ?? new MctsPolicyValueSearch(policyValueNet); + _mulligan = mulligan; + } + + public string Name { get; } + public IEvaluator Evaluator { get; } + public IPolicyValueNet PolicyValueNet { get; } + + public List Mulligan(Game game, Controller controller) + { + return _mulligan(game, controller); + } + + public BotTurnDecision SelectTurn(Game game, Controller controller, BotTurnContext context) + { + return _search.Search(game, controller.Id, Evaluator, context.OpponentPolicy, context.SearchConfig); + } + } + + public static class FirebatNaxxPolicies + { + public static IBotPolicy BaselineDruid() + { + return new BaselinePolicy("baseline-combo-druid", new ComboDruidScore()); + } + + public static IBotPolicy BaselineZoo() + { + return new BaselinePolicy("baseline-zoo-warlock", new ZooWarlockScore()); + } + + public static IBotPolicy RandomDruid(int seed) + { + return new RandomActionPolicy("random-combo-druid", seed); + } + + public static IBotPolicy RandomZoo(int seed) + { + return new RandomActionPolicy("random-zoo-warlock", seed); + } + + public static IBotPolicy CandidateDruid() + { + return new SearchPolicy( + "candidate-combo-druid", + new FirebatNaxxDruidEvaluator(), + new AlternatingTurnSearch(), + (game, controller) => controller.Choice == null + ? new List() + : new ComboDruidScore().MulliganRule().Invoke(controller.Choice.Choices.Select(id => game.IdEntityDic[id]).ToList())); + } + + public static IBotPolicy CandidateZoo() + { + return new SearchPolicy( + "candidate-zoo-warlock", + new FirebatNaxxZooEvaluator(), + new AlternatingTurnSearch(), + (game, controller) => controller.Choice == null + ? new List() + : new ZooWarlockScore().MulliganRule().Invoke(controller.Choice.Choices.Select(id => game.IdEntityDic[id]).ToList())); + } + + public static IPolicyValueBotPolicy PolicyValueDruid(bool fullInformation = false) + { + IPolicyValueNet net = new FirebatNaxxPolicyValueNet(FirebatNaxxSide.ComboDruid, fullInformation); + return new PolicyValuePolicy( + fullInformation ? "teacher-full-info-combo-druid" : "policy-value-combo-druid", + net, + (game, controller) => controller.Choice == null + ? new List() + : new ComboDruidScore().MulliganRule().Invoke(controller.Choice.Choices.Select(id => game.IdEntityDic[id]).ToList())); + } + + public static IPolicyValueBotPolicy PolicyValueZoo(bool fullInformation = false) + { + IPolicyValueNet net = new FirebatNaxxPolicyValueNet(FirebatNaxxSide.ZooWarlock, fullInformation); + return new PolicyValuePolicy( + fullInformation ? "teacher-full-info-zoo-warlock" : "policy-value-zoo-warlock", + net, + (game, controller) => controller.Choice == null + ? new List() + : new ZooWarlockScore().MulliganRule().Invoke(controller.Choice.Choices.Select(id => game.IdEntityDic[id]).ToList())); + } + + public static IPolicyValueBotPolicy TrainedPolicyValueDruid(string modelPath, bool fullInformation = false) + { + IPolicyValueNet net = new LinearPolicyValueNet(modelPath, FirebatNaxxSide.ComboDruid, fullInformation); + return new PolicyValuePolicy( + fullInformation ? "trained-full-info-combo-druid" : "trained-policy-value-combo-druid", + net, + (game, controller) => controller.Choice == null + ? new List() + : new ComboDruidScore().MulliganRule().Invoke(controller.Choice.Choices.Select(id => game.IdEntityDic[id]).ToList())); + } + + public static IPolicyValueBotPolicy TrainedPolicyValueZoo(string modelPath, bool fullInformation = false) + { + IPolicyValueNet net = new LinearPolicyValueNet(modelPath, FirebatNaxxSide.ZooWarlock, fullInformation); + return new PolicyValuePolicy( + fullInformation ? "trained-full-info-zoo-warlock" : "trained-policy-value-zoo-warlock", + net, + (game, controller) => controller.Choice == null + ? new List() + : new ZooWarlockScore().MulliganRule().Invoke(controller.Choice.Choices.Select(id => game.IdEntityDic[id]).ToList())); + } + + public static IPolicyValueBotPolicy PolicyValueLineSearchDruid(bool fullInformation = false) + { + IPolicyValueNet net = new FirebatNaxxPolicyValueNet(FirebatNaxxSide.ComboDruid, fullInformation); + return new PolicyValuePolicy( + fullInformation ? "teacher-full-info-lines-combo-druid" : "policy-value-lines-combo-druid", + net, + (game, controller) => controller.Choice == null + ? new List() + : new ComboDruidScore().MulliganRule().Invoke(controller.Choice.Choices.Select(id => game.IdEntityDic[id]).ToList()), + new PolicyValueSearch(net)); + } + + public static IPolicyValueBotPolicy PolicyValueLineSearchZoo(bool fullInformation = false) + { + IPolicyValueNet net = new FirebatNaxxPolicyValueNet(FirebatNaxxSide.ZooWarlock, fullInformation); + return new PolicyValuePolicy( + fullInformation ? "teacher-full-info-lines-zoo-warlock" : "policy-value-lines-zoo-warlock", + net, + (game, controller) => controller.Choice == null + ? new List() + : new ZooWarlockScore().MulliganRule().Invoke(controller.Choice.Choices.Select(id => game.IdEntityDic[id]).ToList()), + new PolicyValueSearch(net)); + } + } + + internal sealed class FirebatNaxxPolicySet + { + public FirebatNaxxPolicySet(string name, IBotPolicy druid, IBotPolicy zoo) + { + Name = name; + Druid = druid; + Zoo = zoo; + } + + public string Name { get; } + public IBotPolicy Druid { get; } + public IBotPolicy Zoo { get; } + } + + internal static class FirebatNaxxPolicyFactory + { + public static FirebatNaxxPolicySet Create(string policy, string weightsPath, string namePrefix, long randomSeed = 1) + { + string normalized = (policy ?? "baseline").Trim().ToLowerInvariant(); + switch (normalized) + { + case "random": + case "random-actions": + return new FirebatNaxxPolicySet($"{namePrefix}-random", FirebatNaxxPolicies.RandomDruid(MixSeed(randomSeed, 17)), FirebatNaxxPolicies.RandomZoo(MixSeed(randomSeed, 29))); + case "baseline": + case "heuristic": + return new FirebatNaxxPolicySet($"{namePrefix}-baseline", FirebatNaxxPolicies.BaselineDruid(), FirebatNaxxPolicies.BaselineZoo()); + case "candidate": + case "search": + return new FirebatNaxxPolicySet($"{namePrefix}-candidate", FirebatNaxxPolicies.CandidateDruid(), FirebatNaxxPolicies.CandidateZoo()); + case "pv": + case "policy-value": + case "mcts": + case "mcts-pv": + case "mcts-policy-value": + return new FirebatNaxxPolicySet($"{namePrefix}-policy-value", FirebatNaxxPolicies.PolicyValueDruid(), FirebatNaxxPolicies.PolicyValueZoo()); + case "pv-lines": + case "policy-value-lines": + return new FirebatNaxxPolicySet($"{namePrefix}-policy-value-lines", FirebatNaxxPolicies.PolicyValueLineSearchDruid(), FirebatNaxxPolicies.PolicyValueLineSearchZoo()); + case "full-info-pv": + case "full-information-pv": + case "full-info-policy-value": + return new FirebatNaxxPolicySet($"{namePrefix}-full-info-policy-value", FirebatNaxxPolicies.PolicyValueDruid(fullInformation: true), FirebatNaxxPolicies.PolicyValueZoo(fullInformation: true)); + case "trained-pv": + case "trained-policy-value": + if (String.IsNullOrWhiteSpace(weightsPath)) + throw new ArgumentException("A weights path is required for trained policy-value.", nameof(weightsPath)); + return new FirebatNaxxPolicySet($"{namePrefix}-trained-policy-value", FirebatNaxxPolicies.TrainedPolicyValueDruid(weightsPath), FirebatNaxxPolicies.TrainedPolicyValueZoo(weightsPath)); + case "trained-full-info-pv": + case "trained-full-information-pv": + case "trained-full-info-policy-value": + if (String.IsNullOrWhiteSpace(weightsPath)) + throw new ArgumentException("A weights path is required for trained full-information policy-value.", nameof(weightsPath)); + return new FirebatNaxxPolicySet($"{namePrefix}-trained-full-info-policy-value", FirebatNaxxPolicies.TrainedPolicyValueDruid(weightsPath, fullInformation: true), FirebatNaxxPolicies.TrainedPolicyValueZoo(weightsPath, fullInformation: true)); + default: + throw new ArgumentException($"Unknown policy '{policy}'. Use random, baseline, candidate, pv, full-info-pv, trained-pv, or trained-full-info-pv.", nameof(policy)); + } + } + + private static int MixSeed(long seed, int salt) + { + return unchecked((int)(seed * 1103515245 + 12345 + salt * 1000003)); + } + } +} diff --git a/core-extensions/SabberStoneBasicAI/src/Bots/FirebatNaxxTerminalViewer.cs b/core-extensions/SabberStoneBasicAI/src/Bots/FirebatNaxxTerminalViewer.cs new file mode 100644 index 000000000..3789e2d33 --- /dev/null +++ b/core-extensions/SabberStoneBasicAI/src/Bots/FirebatNaxxTerminalViewer.cs @@ -0,0 +1,229 @@ +#region copyright +// SabberStone, Hearthstone Simulator in C# .NET Core +// Copyright (C) 2017-2019 SabberStone Team, darkfriend77 & rnilva +// +// SabberStone is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License. +// SabberStone is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +#endregion +using System; +using System.IO; +using System.Linq; +using SabberStoneCore.Enums; +using SabberStoneCore.Model; +using SabberStoneCore.Model.Entities; +using SabberStoneCore.Tasks.PlayerTasks; + +namespace SabberStoneBasicAI.Bots +{ + public sealed class FirebatNaxxTerminalViewerConfig + { + public long Seed { get; set; } = 1; + public int MaxTurns { get; set; } = 80; + public int MaxDecisions { get; set; } = 1000; + public string Policy { get; set; } = "candidate"; + public string WeightsPath { get; set; } + public string EvaluationPolicy { get; set; } + public string EvaluationWeightsPath { get; set; } + public bool ShowEvaluations { get; set; } + public int EvaluationDecision { get; set; } = -1; + public int MaxEvaluationFeatures { get; set; } + public SearchConfig SearchConfig { get; set; } = new SearchConfig(); + public bool RenderAfterEachAction { get; set; } + } + + public sealed class FirebatNaxxTerminalViewerResult + { + public PlayState DruidResult { get; set; } + public PlayState ZooResult { get; set; } + public int Turns { get; set; } + public int Decisions { get; set; } + public int Actions { get; set; } + public int NodesVisited { get; set; } + public TimeSpan SearchElapsed { get; set; } + public bool HitDecisionCap { get; set; } + public string Error { get; set; } + public double AverageMoveMs => Decisions == 0 ? 0 : SearchElapsed.TotalMilliseconds / Decisions; + } + + public static class FirebatNaxxTerminalViewer + { + public static FirebatNaxxTerminalViewerResult Show(FirebatNaxxTerminalViewerConfig config, TextWriter writer) + { + if (config == null) + throw new ArgumentNullException(nameof(config)); + if (writer == null) + throw new ArgumentNullException(nameof(writer)); + if (config.SearchConfig == null) + config.SearchConfig = new SearchConfig(); + + var result = new FirebatNaxxTerminalViewerResult + { + DruidResult = PlayState.INVALID, + ZooResult = PlayState.INVALID + }; + + try + { + FirebatNaxxPolicySet policies = FirebatNaxxPolicyFactory.Create(config.Policy, config.WeightsPath, "viewer", config.Seed); + FirebatNaxxPolicySet evaluationPolicies = CreateEvaluationPolicies(config); + Game game = FirebatNaxxGameFactory.Create(config.Seed); + game.StartGame(); + + writer.WriteLine($"Firebat Naxx game viewer"); + writer.WriteLine($"seed={config.Seed}, policy={policies.Name}"); + writer.WriteLine($"druid={policies.Druid.Name}, zoo={policies.Zoo.Name}"); + if (config.ShowEvaluations || config.EvaluationDecision > 0) + writer.WriteLine($"evalPolicy={evaluationPolicies.Name}"); + writer.WriteLine(); + + PrintMulligan(game, game.Player1, policies.Druid, writer); + PrintMulligan(game, game.Player2, policies.Zoo, writer); + game.MainReady(); + + Render(game, writer, "Initial state"); + + while (game.State == State.RUNNING && game.Turn <= config.MaxTurns && result.Decisions < config.MaxDecisions) + { + Controller player = game.CurrentPlayer; + IBotPolicy currentPolicy = player == game.Player1 ? policies.Druid : policies.Zoo; + IBotPolicy opponentPolicy = player == game.Player1 ? policies.Zoo : policies.Druid; + int currentPlayerId = player.Id; + int decisionNumber = result.Decisions + 1; + + if (config.EvaluationDecision == decisionNumber) + { + Render(game, writer, $"Evaluation spot before decision {decisionNumber}"); + PrintEvaluations(game, evaluationPolicies, config.MaxEvaluationFeatures, writer); + break; + } + + writer.WriteLine(); + writer.WriteLine($"=== Turn {game.Turn}: {player.Name} ({currentPolicy.Name}) ==="); + if (config.ShowEvaluations) + PrintEvaluations(game, evaluationPolicies, config.MaxEvaluationFeatures, writer); + BotTurnDecision decision = currentPolicy.SelectTurn(game, player, new BotTurnContext(config.SearchConfig, opponentPolicy)); + result.Decisions++; + result.NodesVisited += decision.NodesVisited; + result.SearchElapsed += decision.Elapsed; + writer.WriteLine($"decision score={decision.Score}, search={decision.SearchName}, nodes={decision.NodesVisited}, elapsedMs={decision.Elapsed.TotalMilliseconds:0.0}"); + + if (!ProcessDecision(game, currentPlayerId, decision, config.RenderAfterEachAction, writer, result)) + break; + + if (!config.RenderAfterEachAction && game.State == State.RUNNING) + Render(game, writer, "After decision"); + } + + result.HitDecisionCap = game.State == State.RUNNING && result.Decisions >= config.MaxDecisions; + result.DruidResult = game.Player1.PlayState; + result.ZooResult = game.Player2.PlayState; + result.Turns = game.Turn; + + Render(game, writer, "Final state"); + writer.WriteLine(); + writer.WriteLine($"Result: {FirebatNaxxGameFactory.DruidName}={game.Player1.PlayState}, {FirebatNaxxGameFactory.WarlockName}={game.Player2.PlayState}, turns={game.Turn}, decisions={result.Decisions}, actions={result.Actions}, avgMoveMs={result.AverageMoveMs:0.0}, nodes={result.NodesVisited}"); + if (result.HitDecisionCap) + writer.WriteLine($"Stopped after max decisions ({config.MaxDecisions})."); + } + catch (Exception ex) + { + result.Error = ex.GetType().Name + ": " + ex.Message; + writer.WriteLine(result.Error); + } + + return result; + } + + private static void PrintMulligan(Game game, Controller controller, IBotPolicy policy, TextWriter writer) + { + if (controller.Choice == null) + return; + + var choices = controller.Choice.Choices + .Select(id => game.IdEntityDic[id]) + .OfType() + .ToList(); + var keepIds = policy.Mulligan(game, controller); + string keep = String.Join(", ", choices.Where(card => keepIds.Contains(card.Id)).Select(CardName)); + string replace = String.Join(", ", choices.Where(card => !keepIds.Contains(card.Id)).Select(CardName)); + + writer.WriteLine($"{controller.Name} opening hand: {String.Join(", ", choices.Select(CardName))}"); + writer.WriteLine($"{controller.Name} keep: {(String.IsNullOrEmpty(keep) ? "(none)" : keep)}"); + writer.WriteLine($"{controller.Name} mulligan: {(String.IsNullOrEmpty(replace) ? "(none)" : replace)}"); + game.Process(ChooseTask.Mulligan(controller, keepIds)); + } + + private static FirebatNaxxPolicySet CreateEvaluationPolicies(FirebatNaxxTerminalViewerConfig config) + { + string policy = String.IsNullOrWhiteSpace(config.EvaluationPolicy) ? config.Policy : config.EvaluationPolicy; + string weights = String.IsNullOrWhiteSpace(config.EvaluationWeightsPath) ? config.WeightsPath : config.EvaluationWeightsPath; + return FirebatNaxxPolicyFactory.Create(policy, weights, "eval", config.Seed); + } + + private static void PrintEvaluations(Game game, FirebatNaxxPolicySet policies, int maxFeatures, TextWriter writer) + { + PrintEvaluation(game, game.Player1, policies.Druid.Evaluator, Math.Max(0, maxFeatures), writer); + PrintEvaluation(game, game.Player2, policies.Zoo.Evaluator, Math.Max(0, maxFeatures), writer); + } + + private static void PrintEvaluation(Game game, Controller controller, IEvaluator evaluator, int maxFeatures, TextWriter writer) + { + EvaluationResult eval = evaluator.Evaluate(game, controller.Id); + writer.WriteLine($"eval {controller.Name}: {evaluator.Name}, score={eval.Score}, value={eval.Score / 100000.0:0.000}"); + foreach (var feature in eval.Features + .Where(kvp => kvp.Value != 0) + .OrderByDescending(kvp => Math.Abs(kvp.Value)) + .ThenBy(kvp => kvp.Key, StringComparer.Ordinal) + .Take(maxFeatures)) + { + writer.WriteLine($" {feature.Key}: {feature.Value}"); + } + } + + private static bool ProcessDecision(Game game, int playerId, BotTurnDecision decision, bool renderAfterEachAction, TextWriter writer, FirebatNaxxTerminalViewerResult result) + { + var tasks = decision.Tasks; + if (tasks.Count == 0) + tasks = game.CurrentPlayer.Options().Where(task => !(task is ConcedeTask)).Take(1).ToList(); + + int index = 1; + foreach (PlayerTask task in tasks) + { + if (game.State != State.RUNNING || game.CurrentPlayer.Id != playerId) + break; + + writer.WriteLine($" {index}. {task.FullPrint()}"); + if (!game.Process(task)) + { + writer.WriteLine(" action rejected by engine"); + return false; + } + + result.Actions++; + if (renderAfterEachAction) + Render(game, writer, $"After action {index}"); + index++; + } + + return true; + } + + private static void Render(Game game, TextWriter writer, string title) + { + writer.WriteLine(); + writer.WriteLine($"--- {title} ---"); + writer.Write(SabberStoneCore.Visualizer.Visualizer.Visualize(game)); + } + + private static string CardName(IPlayable playable) + { + return playable?.Card?.Name ?? "(unknown)"; + } + } +} diff --git a/core-extensions/SabberStoneBasicAI/src/Bots/LinearPolicyValueNet.cs b/core-extensions/SabberStoneBasicAI/src/Bots/LinearPolicyValueNet.cs new file mode 100644 index 000000000..72162aadc --- /dev/null +++ b/core-extensions/SabberStoneBasicAI/src/Bots/LinearPolicyValueNet.cs @@ -0,0 +1,514 @@ +#region copyright +// SabberStone, Hearthstone Simulator in C# .NET Core +// Copyright (C) 2017-2019 SabberStone Team, darkfriend77 & rnilva +// +// SabberStone is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License. +// SabberStone is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +#endregion +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Json; +using SabberStoneBasicAI.Meta; +using SabberStoneCore.Enums; +using SabberStoneCore.Model; +using SabberStoneCore.Model.Entities; +using SabberStoneCore.Tasks.PlayerTasks; + +namespace SabberStoneBasicAI.Bots +{ + public sealed class LinearPolicyValueNet : IPolicyValueNet + { + private readonly LinearValueModel _model; + private readonly LinearPolicyModel _policyModel; + private readonly bool _fullInformation; + + public LinearPolicyValueNet(string path, FirebatNaxxSide side, bool fullInformation = false) + { + LinearValueModelBundle bundle = LinearValueModelBundle.Load(path); + string sideName = SideName(side); + _model = bundle.Find(fullInformation ? "full_info_value" : "legal_value", sideName); + _policyModel = bundle.FindPolicy("legal_policy", sideName); + _fullInformation = fullInformation; + Name = $"linear-{_model.Kind}-{_model.Side}-policy-value-net"; + } + + public string Name { get; } + + public PolicyValuePrediction Predict(Game game, int playerId, IReadOnlyList legalActions) + { + Dictionary featureMap = FirebatNaxxFeatureExtractor.Extract(game, playerId, _fullInformation); + double value = _model.Evaluate(featureMap); + return new PolicyValuePrediction(value, BuildPriors(game, playerId, legalActions), featureMap); + } + + private Dictionary BuildPriors(Game game, int playerId, IReadOnlyList legalActions) + { + if (_policyModel == null) + return UniformPriors(legalActions); + if (legalActions == null || legalActions.Count == 0) + return new Dictionary(); + + var logits = new List>(); + double maxLogit = double.NegativeInfinity; + foreach (PlayerTask action in legalActions) + { + string key = FirebatNaxxPolicyValueNet.ActionKey(action); + double logit = _policyModel.Evaluate(FirebatNaxxActionFeatureExtractor.Extract(game, playerId, action)); + logits.Add(new KeyValuePair(key, logit)); + if (logit > maxLogit) + maxLogit = logit; + } + + double total = 0; + var raw = new Dictionary(); + foreach (KeyValuePair entry in logits) + { + double value = Math.Exp(Math.Max(-40.0, entry.Value - maxLogit)); + raw[entry.Key] = value; + total += value; + } + + if (total <= 0) + return UniformPriors(legalActions); + return raw.ToDictionary(kvp => kvp.Key, kvp => kvp.Value / total); + } + + private static Dictionary UniformPriors(IReadOnlyList legalActions) + { + if (legalActions == null || legalActions.Count == 0) + return new Dictionary(); + + double prior = 1.0 / legalActions.Count; + var result = new Dictionary(); + foreach (PlayerTask action in legalActions) + result[FirebatNaxxPolicyValueNet.ActionKey(action)] = prior; + return result; + } + + private static string SideName(FirebatNaxxSide side) + { + return side == FirebatNaxxSide.ComboDruid ? "combo_druid" : "zoo_warlock"; + } + } + + public sealed class LinearValueModel + { + public LinearValueModel(string kind, string side, IReadOnlyList featureSchema, IReadOnlyList weights) + { + Kind = kind; + Side = side; + FeatureSchema = featureSchema; + Weights = weights; + } + + public string Kind { get; } + public string Side { get; } + public IReadOnlyList FeatureSchema { get; } + public IReadOnlyList Weights { get; } + + public double Evaluate(IReadOnlyDictionary features) + { + double score = 0; + for (int i = 0; i < FeatureSchema.Count && i < Weights.Count; i++) + { + if (features.TryGetValue(FeatureSchema[i], out double value)) + score += value * Weights[i]; + } + return Math.Tanh(score); + } + } + + public sealed class LinearValueModelBundle + { + private readonly List _models; + private readonly List _policyModels; + + private LinearValueModelBundle(List models, List policyModels) + { + _models = models; + _policyModels = policyModels; + } + + public static LinearValueModelBundle Load(string path) + { + if (String.IsNullOrWhiteSpace(path)) + throw new ArgumentException("Model path is required.", nameof(path)); + if (!File.Exists(path)) + throw new FileNotFoundException("Value model file not found.", path); + + var serializer = new DataContractJsonSerializer(typeof(LinearValueModelFile)); + using (FileStream stream = File.OpenRead(path)) + { + var file = (LinearValueModelFile)serializer.ReadObject(stream); + if (file == null || file.FeatureSchema == null || file.Models == null) + throw new InvalidOperationException("Invalid linear value model file."); + + var models = file.Models.Select(model => + { + if (model.Weights == null || model.Weights.Length != file.FeatureSchema.Length) + throw new InvalidOperationException($"Model {model.Kind}:{model.Side} has {model.Weights?.Length ?? 0} weights for {file.FeatureSchema.Length} features."); + return new LinearValueModel(model.Kind, model.Side, file.FeatureSchema, model.Weights); + }).ToList(); + + var policyModels = new List(); + if (file.PolicyModels != null && file.PolicyModels.Length > 0) + { + if (file.ActionFeatureSchema == null || file.ActionFeatureSchema.Length == 0) + throw new InvalidOperationException("Policy models require actionFeatureSchema."); + policyModels = file.PolicyModels.Select(model => + { + if (model.Weights == null || model.Weights.Length != file.ActionFeatureSchema.Length) + throw new InvalidOperationException($"Policy model {model.Kind}:{model.Side} has {model.Weights?.Length ?? 0} weights for {file.ActionFeatureSchema.Length} action features."); + return new LinearPolicyModel(model.Kind, model.Side, file.ActionFeatureSchema, model.Weights); + }).ToList(); + } + + return new LinearValueModelBundle(models, policyModels); + } + } + + public LinearValueModel Find(string kind, string side) + { + LinearValueModel model = _models.FirstOrDefault(m => m.Kind == kind && m.Side == side); + if (model == null) + throw new InvalidOperationException($"Model file does not contain {kind}:{side}."); + return model; + } + + public LinearPolicyModel FindPolicy(string kind, string side) + { + return _policyModels.FirstOrDefault(m => m.Kind == kind && m.Side == side); + } + } + + public sealed class LinearPolicyModel + { + public LinearPolicyModel(string kind, string side, IReadOnlyList actionFeatureSchema, IReadOnlyList weights) + { + Kind = kind; + Side = side; + ActionFeatureSchema = actionFeatureSchema; + Weights = weights; + } + + public string Kind { get; } + public string Side { get; } + public IReadOnlyList ActionFeatureSchema { get; } + public IReadOnlyList Weights { get; } + + public double Evaluate(IReadOnlyDictionary features) + { + double score = 0; + for (int i = 0; i < ActionFeatureSchema.Count && i < Weights.Count; i++) + { + if (features.TryGetValue(ActionFeatureSchema[i], out double value)) + score += value * Weights[i]; + } + return score; + } + } + + [DataContract] + internal sealed class LinearValueModelFile + { + [DataMember(Name = "schema")] + public string Schema { get; set; } + + [DataMember(Name = "featureSchema")] + public string[] FeatureSchema { get; set; } + + [DataMember(Name = "actionFeatureSchema")] + public string[] ActionFeatureSchema { get; set; } + + [DataMember(Name = "models")] + public LinearValueModelRecord[] Models { get; set; } + + [DataMember(Name = "policyModels")] + public LinearValueModelRecord[] PolicyModels { get; set; } + } + + [DataContract] + internal sealed class LinearValueModelRecord + { + [DataMember(Name = "kind")] + public string Kind { get; set; } + + [DataMember(Name = "side")] + public string Side { get; set; } + + [DataMember(Name = "weights")] + public double[] Weights { get; set; } + } + + public static class FirebatNaxxActionFeatureExtractor + { + private static readonly string[] CardIds = Decks.FirebatNaxxComboDruid + .Concat(Decks.FirebatNaxxZooWarlock) + .Select(card => card.Id) + .Concat(new[] { "GAME_005" }) + .Distinct() + .OrderBy(id => id, StringComparer.Ordinal) + .ToArray(); + + public static Dictionary Extract(Game game, int playerId, PlayerTask action) + { + Controller viewer = game.ControllerById(playerId); + var features = new Dictionary + { + ["bias"] = 1.0, + ["turn_norm"] = game.Turn / 20.0, + ["viewer_is_combo_druid"] = viewer.HeroClass == CardClass.DRUID ? 1.0 : 0.0, + ["viewer_is_zoo_warlock"] = viewer.HeroClass == CardClass.WARLOCK ? 1.0 : 0.0, + ["remaining_mana_norm"] = viewer.RemainingMana / 10.0, + ["base_mana_norm"] = viewer.BaseMana / 10.0, + ["board_count_norm"] = viewer.BoardZone.Count / 7.0, + ["opponent_board_count_norm"] = viewer.Opponent.BoardZone.Count / 7.0, + [$"type_{action.PlayerTaskType.ToString().ToLowerInvariant()}"] = 1.0 + }; + + AddSourceFeatures(features, viewer, action); + AddTargetFeatures(features, viewer, action); + if (action.ChooseOne == 1) + features["choose_one_1"] = 1.0; + if (action.ChooseOne == 2) + features["choose_one_2"] = 1.0; + if (action is EndTurnTask) + features["end_turn_remaining_mana_norm"] = viewer.RemainingMana / 10.0; + + return features; + } + + private static void AddSourceFeatures(Dictionary features, Controller viewer, PlayerTask action) + { + if (action.Source == null) + return; + + features[$"source_card_{action.Source.Card.Id}"] = 1.0; + features[$"source_type_{action.Source.Card.Type.ToString().ToLowerInvariant()}"] = 1.0; + features["source_cost_norm"] = action.Source.Cost / 10.0; + features["remaining_mana_after_source_cost_norm"] = Math.Max(0, viewer.RemainingMana - action.Source.Cost) / 10.0; + if (action.Source.Card.Cost == 0) + features["source_zero_cost"] = 1.0; + if (action.Source is Minion minion) + { + features["source_attack_norm"] = Math.Max(0, minion.AttackDamage) / 12.0; + features["source_health_norm"] = Math.Max(0, minion.Health) / 12.0; + if (minion.HasTaunt) + features["source_taunt"] = 1.0; + if (minion.HasCharge) + features["source_charge"] = 1.0; + } + } + + private static void AddTargetFeatures(Dictionary features, Controller viewer, PlayerTask action) + { + if (action.Target == null) + { + features["target_none"] = 1.0; + return; + } + + features["target_any"] = 1.0; + if (action.Target.Controller == viewer) + features["target_friendly"] = 1.0; + else + features["target_enemy"] = 1.0; + + if (action.Target == viewer.Opponent.Hero) + features["target_enemy_hero"] = 1.0; + else if (action.Target == viewer.Hero) + features["target_friendly_hero"] = 1.0; + else + features["target_minion"] = 1.0; + + if (action.Target is Minion target) + { + features[$"target_card_{target.Card.Id}"] = 1.0; + features["target_attack_norm"] = Math.Max(0, target.AttackDamage) / 12.0; + features["target_health_norm"] = Math.Max(0, target.Health) / 12.0; + if (target.HasTaunt) + features["target_taunt"] = 1.0; + if (target.Damage > 0) + features["target_damaged"] = 1.0; + } + } + + public static IReadOnlyList FeatureSchema { get; } = BuildFeatureSchema(); + + private static IReadOnlyList BuildFeatureSchema() + { + var features = new List + { + "bias", + "turn_norm", + "viewer_is_combo_druid", + "viewer_is_zoo_warlock", + "remaining_mana_norm", + "base_mana_norm", + "board_count_norm", + "opponent_board_count_norm", + "type_choose", + "type_end_turn", + "type_hero_attack", + "type_hero_power", + "type_minion_attack", + "type_play_card", + "target_none", + "target_any", + "target_friendly", + "target_enemy", + "target_enemy_hero", + "target_friendly_hero", + "target_minion", + "target_attack_norm", + "target_health_norm", + "target_taunt", + "target_damaged", + "source_type_minion", + "source_type_spell", + "source_type_weapon", + "source_cost_norm", + "source_attack_norm", + "source_health_norm", + "source_zero_cost", + "source_taunt", + "source_charge", + "remaining_mana_after_source_cost_norm", + "end_turn_remaining_mana_norm", + "choose_one_1", + "choose_one_2" + }; + + foreach (string cardId in CardIds) + features.Add($"source_card_{cardId}"); + foreach (string cardId in CardIds) + features.Add($"target_card_{cardId}"); + return features; + } + } + + public static class FirebatNaxxFeatureExtractor + { + private static readonly string[] CardIds = Decks.FirebatNaxxComboDruid + .Concat(Decks.FirebatNaxxZooWarlock) + .Select(card => card.Id) + .Concat(new[] { "GAME_005" }) + .Distinct() + .OrderBy(id => id, StringComparer.Ordinal) + .ToArray(); + + public static IReadOnlyList FeatureSchema { get; } = BuildFeatureSchema(); + + public static Dictionary Extract(Game game, int playerId, bool fullInformation) + { + Controller viewer = game.ControllerById(playerId); + Controller opponent = viewer.Opponent; + var features = new Dictionary + { + ["bias"] = 1.0, + ["turn_norm"] = game.Turn / 20.0, + ["current_player_is_viewer"] = game.CurrentPlayer.Id == viewer.Id ? 1.0 : 0.0, + ["viewer_is_combo_druid"] = viewer.HeroClass == CardClass.DRUID ? 1.0 : 0.0, + ["viewer_is_zoo_warlock"] = viewer.HeroClass == CardClass.WARLOCK ? 1.0 : 0.0 + }; + + AddControllerFeatures(features, "viewer", viewer, revealHand: true, revealDeck: true); + AddControllerFeatures(features, "opponent", opponent, revealHand: fullInformation, revealDeck: fullInformation); + features["hero_health_delta_norm"] = (viewer.Hero.Health + viewer.Hero.Armor - opponent.Hero.Health - opponent.Hero.Armor) / 30.0; + features["board_attack_delta_norm"] = (BoardAttack(viewer) - BoardAttack(opponent)) / 30.0; + features["board_health_delta_norm"] = (BoardHealth(viewer) - BoardHealth(opponent)) / 60.0; + + foreach (string name in FeatureSchema) + if (!features.ContainsKey(name)) + features[name] = 0.0; + + return features; + } + + private static IReadOnlyList BuildFeatureSchema() + { + var features = new List + { + "bias", + "turn_norm", + "current_player_is_viewer", + "viewer_is_combo_druid", + "viewer_is_zoo_warlock" + }; + + AddControllerSchema(features, "viewer"); + AddControllerSchema(features, "opponent"); + features.Add("hero_health_delta_norm"); + features.Add("board_attack_delta_norm"); + features.Add("board_health_delta_norm"); + + foreach (string prefix in new[] { "viewer", "opponent" }) + foreach (string zone in new[] { "hand", "deck", "board", "graveyard" }) + foreach (string cardId in CardIds) + features.Add($"{prefix}_{zone}_{cardId}"); + + return features; + } + + private static void AddControllerSchema(List features, string prefix) + { + features.Add($"{prefix}_hero_health_norm"); + features.Add($"{prefix}_hero_armor_norm"); + features.Add($"{prefix}_base_mana_norm"); + features.Add($"{prefix}_remaining_mana_norm"); + features.Add($"{prefix}_hand_count_norm"); + features.Add($"{prefix}_deck_count_norm"); + features.Add($"{prefix}_board_count_norm"); + features.Add($"{prefix}_board_attack_norm"); + features.Add($"{prefix}_board_health_norm"); + features.Add($"{prefix}_taunt_count_norm"); + } + + private static void AddControllerFeatures(Dictionary features, string prefix, Controller controller, bool revealHand, bool revealDeck) + { + features[$"{prefix}_hero_health_norm"] = controller.Hero.Health / 30.0; + features[$"{prefix}_hero_armor_norm"] = controller.Hero.Armor / 30.0; + features[$"{prefix}_base_mana_norm"] = controller.BaseMana / 10.0; + features[$"{prefix}_remaining_mana_norm"] = controller.RemainingMana / 10.0; + features[$"{prefix}_hand_count_norm"] = controller.HandZone.Count / 10.0; + features[$"{prefix}_deck_count_norm"] = controller.DeckZone.Count / 30.0; + features[$"{prefix}_board_count_norm"] = controller.BoardZone.Count / 7.0; + features[$"{prefix}_board_attack_norm"] = BoardAttack(controller) / 30.0; + features[$"{prefix}_board_health_norm"] = BoardHealth(controller) / 60.0; + features[$"{prefix}_taunt_count_norm"] = controller.BoardZone.Count(minion => minion.HasTaunt) / 7.0; + + if (revealHand) + AddCardCounts(features, $"{prefix}_hand", controller.HandZone.Select(card => card.Card.Id), 2.0); + if (revealDeck) + AddCardCounts(features, $"{prefix}_deck", controller.DeckZone.Select(card => card.Card.Id), 2.0); + + AddCardCounts(features, $"{prefix}_board", controller.BoardZone.Select(card => card.Card.Id), 2.0); + AddCardCounts(features, $"{prefix}_graveyard", controller.GraveyardZone.Select(card => card.Card.Id), 2.0); + } + + private static void AddCardCounts(Dictionary features, string prefix, IEnumerable cardIds, double normalizer) + { + foreach (IGrouping group in cardIds.GroupBy(id => id)) + features[$"{prefix}_{group.Key}"] = group.Count() / normalizer; + } + + private static int BoardAttack(Controller controller) + { + return controller.BoardZone.Sum(minion => Math.Max(0, minion.AttackDamage)) + controller.Hero.TotalAttackDamage; + } + + private static int BoardHealth(Controller controller) + { + return controller.BoardZone.Sum(minion => Math.Max(0, minion.Health)); + } + } +} diff --git a/core-extensions/SabberStoneBasicAI/src/Bots/MctsPolicyValueSearch.cs b/core-extensions/SabberStoneBasicAI/src/Bots/MctsPolicyValueSearch.cs new file mode 100644 index 000000000..99068f14f --- /dev/null +++ b/core-extensions/SabberStoneBasicAI/src/Bots/MctsPolicyValueSearch.cs @@ -0,0 +1,222 @@ +#region copyright +// SabberStone, Hearthstone Simulator in C# .NET Core +// Copyright (C) 2017-2019 SabberStone Team, darkfriend77 & rnilva +// +// SabberStone is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License. +// SabberStone is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +#endregion +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using SabberStoneCore.Enums; +using SabberStoneCore.Model; +using SabberStoneCore.Tasks.PlayerTasks; + +namespace SabberStoneBasicAI.Bots +{ + public sealed class MctsPolicyValueSearch : ISearch + { + private readonly IPolicyValueNet _policyValueNet; + + public MctsPolicyValueSearch(IPolicyValueNet policyValueNet) + { + _policyValueNet = policyValueNet; + } + + public string Name => "mcts-policy-value-search"; + + public BotTurnDecision Search(Game game, int playerId, IEvaluator evaluator, IBotPolicy opponentPolicy, SearchConfig config) + { + var watch = Stopwatch.StartNew(); + int nodesVisited = 0; + var root = new MctsNode(game.Clone(resetRandomSeed: false), null, 0); + IPolicyValueNet opponentNet = (opponentPolicy as IPolicyValueBotPolicy)?.PolicyValueNet; + int iterations = Math.Max(1, config.MctsIterations); + int maxDepth = Math.Max(1, config.MaxDepth); + + for (int i = 0; i < iterations && watch.ElapsedMilliseconds < config.TimeBudgetMs; i++) + { + var path = new List { root }; + MctsNode node = root; + int depth = 0; + + while (node.IsExpanded && node.Children.Count > 0 && depth < maxDepth && node.Game.State == State.RUNNING) + { + node = SelectChild(node, playerId, config.MctsExploration); + path.Add(node); + depth++; + } + + double value = Evaluate(node.Game, playerId, evaluator); + if (node.Game.State == State.RUNNING && depth < maxDepth && !node.IsExpanded) + { + Expand(node, playerId, opponentNet, config.MaxWidth, ref nodesVisited); + value = Evaluate(node.Game, playerId, evaluator); + } + + foreach (MctsNode visited in path) + visited.Backup(value); + } + + if (!root.IsExpanded) + Expand(root, playerId, opponentNet, config.MaxWidth, ref nodesVisited); + if (root.Children.Count == 0) + return Fallback(game, watch, nodesVisited); + + MctsNode best = root.Children + .OrderByDescending(child => child.Visits) + .ThenByDescending(child => child.Q) + .First(); + watch.Stop(); + + int score = (int)(best.Q * 100000); + return new BotTurnDecision(new List { best.Action }, score, watch.Elapsed, Name, nodesVisited, VisitTargets(root)); + } + + private static MctsNode SelectChild(MctsNode node, int rootPlayerId, double exploration) + { + bool maximizing = node.Game.CurrentPlayer.Id == rootPlayerId; + double parentVisits = Math.Max(1, node.Visits); + return node.Children + .OrderByDescending(child => + { + double q = child.Visits == 0 ? 0 : child.Q; + double u = Math.Max(0, exploration) * child.Prior * Math.Sqrt(parentVisits) / (1 + child.Visits); + return (maximizing ? q : -q) + u; + }) + .First(); + } + + private void Expand(MctsNode node, int rootPlayerId, IPolicyValueNet opponentNet, int maxWidth, ref int nodesVisited) + { + if (node.IsExpanded) + return; + node.IsExpanded = true; + if (node.Game.State != State.RUNNING) + return; + + List legalActions = node.Game.CurrentPlayer.Options() + .Where(task => !(task is ConcedeTask)) + .ToList(); + if (legalActions.Count == 0) + return; + + IPolicyValueNet actorNet = node.Game.CurrentPlayer.Id == rootPlayerId ? _policyValueNet : opponentNet; + IReadOnlyDictionary priors = actorNet?.Predict(node.Game, node.Game.CurrentPlayer.Id, legalActions).ActionPriors + ?? UniformPriors(legalActions); + foreach (PlayerTask action in OrderActions(legalActions, priors, maxWidth)) + { + Game clone = node.Game.Clone(resetRandomSeed: false); + if (!clone.Process(action)) + continue; + + string key = FirebatNaxxPolicyValueNet.ActionKey(action); + double prior = priors.TryGetValue(key, out double p) ? p : 0; + node.Children.Add(new MctsNode(clone, action, Math.Max(0, prior))); + nodesVisited++; + } + + NormalizeChildPriors(node); + } + + private static IEnumerable OrderActions(IReadOnlyList legalActions, IReadOnlyDictionary priors, int maxWidth) + { + int limit = maxWidth <= 0 ? legalActions.Count : Math.Max(1, maxWidth); + return legalActions + .OrderByDescending(action => priors.TryGetValue(FirebatNaxxPolicyValueNet.ActionKey(action), out double prior) ? prior : 0) + .Take(limit); + } + + private static void NormalizeChildPriors(MctsNode node) + { + if (node.Children.Count == 0) + return; + + double total = node.Children.Sum(child => child.Prior); + if (total <= 0) + { + double uniform = 1.0 / node.Children.Count; + foreach (MctsNode child in node.Children) + child.Prior = uniform; + return; + } + + foreach (MctsNode child in node.Children) + child.Prior /= total; + } + + private static IReadOnlyDictionary UniformPriors(IReadOnlyList legalActions) + { + if (legalActions.Count == 0) + return new Dictionary(); + double prior = 1.0 / legalActions.Count; + var result = new Dictionary(); + foreach (PlayerTask action in legalActions) + result[FirebatNaxxPolicyValueNet.ActionKey(action)] = prior; + return result; + } + + private static double Evaluate(Game game, int rootPlayerId, IEvaluator evaluator) + { + if (game.State == State.COMPLETE) + { + PlayState result = game.ControllerById(rootPlayerId).PlayState; + if (result == PlayState.WON) + return 1.0; + if (result == PlayState.LOST || result == PlayState.CONCEDED) + return -1.0; + if (result == PlayState.TIED) + return 0.0; + } + + return Math.Max(-1.0, Math.Min(1.0, evaluator.Evaluate(game, rootPlayerId).Score / 100000.0)); + } + + private static IReadOnlyDictionary VisitTargets(MctsNode root) + { + int total = root.Children.Sum(child => child.Visits); + if (total <= 0) + return root.Children.ToDictionary(child => FirebatNaxxPolicyValueNet.ActionKey(child.Action), child => 1.0 / root.Children.Count); + return root.Children.ToDictionary(child => FirebatNaxxPolicyValueNet.ActionKey(child.Action), child => child.Visits / (double)total); + } + + private static BotTurnDecision Fallback(Game game, Stopwatch watch, int nodesVisited) + { + PlayerTask option = game.CurrentPlayer.Options().FirstOrDefault(task => !(task is ConcedeTask)); + watch.Stop(); + return new BotTurnDecision(option == null ? new List() : new List { option }, 0, watch.Elapsed, "mcts-fallback", nodesVisited); + } + + private sealed class MctsNode + { + public MctsNode(Game game, PlayerTask action, double prior) + { + Game = game; + Action = action; + Prior = prior; + } + + public Game Game { get; } + public PlayerTask Action { get; } + public double Prior { get; set; } + public List Children { get; } = new List(); + public bool IsExpanded { get; set; } + public int Visits { get; private set; } + public double ValueSum { get; private set; } + public double Q => Visits == 0 ? 0 : ValueSum / Visits; + + public void Backup(double value) + { + Visits++; + ValueSum += value; + } + } + } +} diff --git a/core-extensions/SabberStoneBasicAI/src/Bots/PolicyValueModels.cs b/core-extensions/SabberStoneBasicAI/src/Bots/PolicyValueModels.cs new file mode 100644 index 000000000..4291ae45c --- /dev/null +++ b/core-extensions/SabberStoneBasicAI/src/Bots/PolicyValueModels.cs @@ -0,0 +1,391 @@ +#region copyright +// SabberStone, Hearthstone Simulator in C# .NET Core +// Copyright (C) 2017-2019 SabberStone Team, darkfriend77 & rnilva +// +// SabberStone is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License. +// SabberStone is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +#endregion +using System; +using System.Collections.Generic; +using System.Linq; +using SabberStoneBasicAI.Meta; +using SabberStoneCore.Enums; +using SabberStoneCore.Model; +using SabberStoneCore.Model.Entities; +using SabberStoneCore.Tasks.PlayerTasks; + +namespace SabberStoneBasicAI.Bots +{ + public enum FirebatNaxxSide + { + ComboDruid, + ZooWarlock + } + + public interface IPolicyValueNet + { + string Name { get; } + PolicyValuePrediction Predict(Game game, int playerId, IReadOnlyList legalActions); + } + + public interface IHiddenStatePredictor + { + string Name { get; } + HiddenStateBelief Predict(Game game, int playerId); + } + + public sealed class PolicyValuePrediction + { + public PolicyValuePrediction(double value, IDictionary actionPriors, IDictionary features) + { + Value = Math.Max(-1.0, Math.Min(1.0, value)); + ActionPriors = actionPriors == null + ? new Dictionary() + : new Dictionary(actionPriors); + Features = features == null + ? new Dictionary() + : new Dictionary(features); + } + + public double Value { get; } + public IReadOnlyDictionary ActionPriors { get; } + public IReadOnlyDictionary Features { get; } + } + + public sealed class HiddenStateBelief + { + public HiddenStateBelief(int viewerPlayerId, int hiddenPlayerId, int observedHandSize, int observedDeckCount, + IDictionary unknownCounts, IDictionary expectedHandCounts, + IReadOnlyList samples) + { + ViewerPlayerId = viewerPlayerId; + HiddenPlayerId = hiddenPlayerId; + ObservedHandSize = observedHandSize; + ObservedDeckCount = observedDeckCount; + UnknownCounts = new Dictionary(unknownCounts); + ExpectedHandCounts = new Dictionary(expectedHandCounts); + Samples = samples; + } + + public int ViewerPlayerId { get; } + public int HiddenPlayerId { get; } + public int ObservedHandSize { get; } + public int ObservedDeckCount { get; } + public IReadOnlyDictionary UnknownCounts { get; } + public IReadOnlyDictionary ExpectedHandCounts { get; } + public IReadOnlyList Samples { get; } + + public double ExpectedHandCount(string cardName) + { + return ExpectedHandCounts + .Where(kvp => Cards.FromId(kvp.Key).Name == cardName) + .Sum(kvp => kvp.Value); + } + } + + public sealed class HiddenStateSample + { + public HiddenStateSample(IReadOnlyList handIds, IReadOnlyList deckIds, double weight) + { + HandIds = handIds; + DeckIds = deckIds; + Weight = weight; + } + + public IReadOnlyList HandIds { get; } + public IReadOnlyList DeckIds { get; } + public double Weight { get; } + } + + public sealed class FirebatNaxxHiddenStatePredictor : IHiddenStatePredictor + { + private readonly int _sampleCount; + + public FirebatNaxxHiddenStatePredictor(int sampleCount = 8) + { + _sampleCount = Math.Max(1, sampleCount); + } + + public string Name => "firebat-naxx-hidden-state-predictor"; + + public HiddenStateBelief Predict(Game game, int playerId) + { + Controller viewer = game.ControllerById(playerId); + Controller hidden = viewer.Opponent; + Dictionary unknownCounts = DeckCountsFor(hidden); + SubtractPublicCards(unknownCounts, hidden); + + int unknownTotal = unknownCounts.Values.Sum(); + int handSize = hidden.HandZone.Count; + var expectedHand = new Dictionary(); + foreach (KeyValuePair kvp in unknownCounts) + expectedHand[kvp.Key] = unknownTotal == 0 ? 0 : kvp.Value * handSize / (double)unknownTotal; + + return new HiddenStateBelief( + viewer.Id, + hidden.Id, + handSize, + hidden.DeckZone.Count, + unknownCounts, + expectedHand, + CreateSamples(game, unknownCounts, handSize)); + } + + private IReadOnlyList CreateSamples(Game game, Dictionary unknownCounts, int handSize) + { + var samples = new List(); + List expanded = Expand(unknownCounts); + if (expanded.Count == 0) + return samples; + + int seed = unchecked(game.Turn * 7919 + (int)game.Step * 104729 + expanded.Count * 31); + for (int i = 0; i < _sampleCount; i++) + { + var rnd = new Random(seed + i * 997); + List pool = expanded.ToList(); + var hand = new List(); + int cards = Math.Min(handSize, pool.Count); + for (int j = 0; j < cards; j++) + { + int index = rnd.Next(pool.Count); + hand.Add(pool[index]); + pool.RemoveAt(index); + } + + samples.Add(new HiddenStateSample(hand, pool, 1.0 / _sampleCount)); + } + + return samples; + } + + private static Dictionary DeckCountsFor(Controller controller) + { + IEnumerable deck = controller.HeroClass == CardClass.DRUID + ? Decks.FirebatNaxxComboDruid + : Decks.FirebatNaxxZooWarlock; + + return deck + .GroupBy(card => card.Id) + .ToDictionary(group => group.Key, group => group.Count()); + } + + private static void SubtractPublicCards(Dictionary counts, Controller hidden) + { + foreach (IPlayable card in hidden.BoardZone.Cast().Concat(hidden.GraveyardZone)) + Subtract(counts, card.Card.Id); + } + + private static void Subtract(Dictionary counts, string cardId) + { + if (!counts.TryGetValue(cardId, out int count)) + return; + if (count <= 1) + counts.Remove(cardId); + else + counts[cardId] = count - 1; + } + + private static List Expand(Dictionary counts) + { + var result = new List(); + foreach (KeyValuePair kvp in counts.OrderBy(kvp => kvp.Key)) + for (int i = 0; i < kvp.Value; i++) + result.Add(kvp.Key); + return result; + } + } + + public sealed class FirebatNaxxPolicyValueNet : IPolicyValueNet + { + private readonly FirebatNaxxSide _side; + private readonly bool _fullInformation; + private readonly IHiddenStatePredictor _hiddenStatePredictor; + private readonly IEvaluator _legalEvaluator; + + public FirebatNaxxPolicyValueNet(FirebatNaxxSide side, bool fullInformation = false, IHiddenStatePredictor hiddenStatePredictor = null) + { + _side = side; + _fullInformation = fullInformation; + _hiddenStatePredictor = hiddenStatePredictor ?? new FirebatNaxxHiddenStatePredictor(); + _legalEvaluator = side == FirebatNaxxSide.ComboDruid + ? (IEvaluator)new FirebatNaxxDruidEvaluator() + : new FirebatNaxxZooEvaluator(); + } + + public string Name => _fullInformation + ? $"full-info-{_side.ToString().ToLowerInvariant()}-policy-value-net" + : $"legal-{_side.ToString().ToLowerInvariant()}-policy-value-net"; + + public PolicyValuePrediction Predict(Game game, int playerId, IReadOnlyList legalActions) + { + Controller c = game.ControllerById(playerId); + Controller op = c.Opponent; + EvaluationResult eval = _legalEvaluator.Evaluate(game, playerId); + var features = eval.Features.ToDictionary(kvp => kvp.Key, kvp => kvp.Value / 1000.0); + + double valueScore = eval.Score / 6500.0; + if (_fullInformation) + valueScore += FullInformationAdjustment(c, op, features); + else + valueScore += BeliefAdjustment(game, playerId, features); + + Dictionary priors = legalActions == null + ? new Dictionary() + : BuildActionPriors(c, legalActions); + + return new PolicyValuePrediction(Math.Tanh(valueScore), priors, features); + } + + private double FullInformationAdjustment(Controller c, Controller op, IDictionary features) + { + double ownReach = _side == FirebatNaxxSide.ComboDruid ? DruidReachFromHand(c) : ZooReachFromHand(c); + double opponentReach = _side == FirebatNaxxSide.ComboDruid ? ZooReachFromHand(op) : DruidReachFromHand(op); + double ownOuts = _side == FirebatNaxxSide.ComboDruid ? CountInDeck(c, "Savage Roar") + CountInDeck(c, "Force of Nature") : CountInDeck(c, "Soulfire") + CountInDeck(c, "Doomguard"); + double opponentOuts = _side == FirebatNaxxSide.ComboDruid ? CountInDeck(op, "Soulfire") + CountInDeck(op, "Doomguard") : CountInDeck(op, "Savage Roar") + CountInDeck(op, "Force of Nature"); + + features["full_info_own_reach"] = ownReach; + features["full_info_opponent_reach"] = opponentReach; + features["full_info_own_outs"] = ownOuts; + features["full_info_opponent_outs"] = opponentOuts; + + return ownReach * 0.10 - opponentReach * 0.13 + ownOuts * 0.04 - opponentOuts * 0.05; + } + + private double BeliefAdjustment(Game game, int playerId, IDictionary features) + { + HiddenStateBelief belief = _hiddenStatePredictor.Predict(game, playerId); + double expectedOpponentReach = _side == FirebatNaxxSide.ComboDruid + ? belief.ExpectedHandCount("Soulfire") * 4 + belief.ExpectedHandCount("Power Overwhelming") * 4 + belief.ExpectedHandCount("Doomguard") * 5 + : belief.ExpectedHandCount("Force of Nature") * 6 + belief.ExpectedHandCount("Savage Roar") * 8; + + features["belief_hidden_hand"] = belief.ObservedHandSize; + features["belief_unknown_cards"] = belief.UnknownCounts.Values.Sum(); + features["belief_expected_opponent_reach"] = expectedOpponentReach; + + return -expectedOpponentReach * 0.08; + } + + private Dictionary BuildActionPriors(Controller c, IReadOnlyList legalActions) + { + var raw = new Dictionary(); + foreach (PlayerTask action in legalActions) + raw[ActionKey(action)] = Math.Exp(ActionLogit(c, action)); + + double total = raw.Values.Sum(); + if (total <= 0) + return raw.ToDictionary(kvp => kvp.Key, kvp => 1.0 / Math.Max(1, raw.Count)); + + return raw.ToDictionary(kvp => kvp.Key, kvp => kvp.Value / total); + } + + private double ActionLogit(Controller c, PlayerTask action) + { + switch (action.PlayerTaskType) + { + case PlayerTaskType.END_TURN: + return -1.8 + c.RemainingMana * -0.10; + case PlayerTaskType.MINION_ATTACK: + case PlayerTaskType.HERO_ATTACK: + return AttackLogit(c, action); + case PlayerTaskType.HERO_POWER: + return _side == FirebatNaxxSide.ZooWarlock ? 0.15 + c.RemainingMana * 0.03 : -0.25; + case PlayerTaskType.PLAY_CARD: + return PlayCardLogit(c, action); + default: + return -2.0; + } + } + + private double AttackLogit(Controller c, PlayerTask action) + { + if (action.Target == c.Opponent.Hero) + return _side == FirebatNaxxSide.ZooWarlock ? 0.85 : 0.35; + return _side == FirebatNaxxSide.ZooWarlock ? 0.25 : 0.50; + } + + private double PlayCardLogit(Controller c, PlayerTask action) + { + if (action.Source == null) + return 0; + + Card card = action.Source.Card; + double curve = Math.Max(0, 4 - card.Cost) * 0.10; + if (_side == FirebatNaxxSide.ComboDruid) + { + if (card.Name == "Wild Growth" && c.BaseMana < 7) + return 1.4; + if (card.Name == "Innervate") + return 0.85; + if (card.Name == "Swipe" || card.Name == "Wrath") + return action.Target == null ? 0.10 : 0.75; + if (card.Name == "Force of Nature" || card.Name == "Savage Roar") + return c.Opponent.Hero.Health <= 16 ? 1.10 : -0.35; + return 0.35 + curve; + } + + if (card.Cost <= Math.Max(1, c.BaseMana)) + return 0.70 + curve; + if (card.Name == "Soulfire" || card.Name == "Power Overwhelming" || card.Name == "Doomguard") + return c.Opponent.Hero.Health <= 12 ? 1.20 : 0.10; + return 0.25 + curve; + } + + public static string ActionKey(PlayerTask action) + { + return action.FullPrint(); + } + + private static int CountInDeck(Controller c, string name) + { + return c.DeckZone.Count(card => card.Card.Name == name); + } + + private static int CountInHand(Controller c, string name) + { + return c.HandZone.Count(card => card.Card.Name == name); + } + + private static int DruidReachFromHand(Controller c) + { + int force = CountInHand(c, "Force of Nature"); + int roars = CountInHand(c, "Savage Roar"); + return force > 0 && roars > 0 ? 6 + roars * 8 : 0; + } + + private static int ZooReachFromHand(Controller c) + { + return CountInHand(c, "Soulfire") * 4 + + CountInHand(c, "Power Overwhelming") * 4 + + CountInHand(c, "Doomguard") * 5 + + CountInHand(c, "Abusive Sergeant") * 2 + + CountInHand(c, "Dark Iron Dwarf") * 2; + } + } + + public sealed class PolicyValueEvaluator : IEvaluator + { + private readonly IPolicyValueNet _net; + + public PolicyValueEvaluator(IPolicyValueNet net) + { + _net = net; + } + + public string Name => _net.Name + " evaluator"; + + public EvaluationResult Evaluate(Game game, int playerId) + { + IReadOnlyList legalActions = game.State == State.RUNNING && game.CurrentPlayer.Id == playerId + ? game.CurrentPlayer.Options().Where(task => !(task is ConcedeTask)).ToList() + : new List(); + PolicyValuePrediction prediction = _net.Predict(game, playerId, legalActions); + return new EvaluationResult((int)(prediction.Value * 100000), prediction.Features.ToDictionary(kvp => kvp.Key, kvp => (int)(kvp.Value * 1000))); + } + } +} diff --git a/core-extensions/SabberStoneBasicAI/src/Bots/PolicyValueSearch.cs b/core-extensions/SabberStoneBasicAI/src/Bots/PolicyValueSearch.cs new file mode 100644 index 000000000..72e0048bd --- /dev/null +++ b/core-extensions/SabberStoneBasicAI/src/Bots/PolicyValueSearch.cs @@ -0,0 +1,247 @@ +#region copyright +// SabberStone, Hearthstone Simulator in C# .NET Core +// Copyright (C) 2017-2019 SabberStone Team, darkfriend77 & rnilva +// +// SabberStone is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License. +// SabberStone is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +#endregion +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using SabberStoneCore.Enums; +using SabberStoneCore.Model; +using SabberStoneCore.Tasks.PlayerTasks; + +namespace SabberStoneBasicAI.Bots +{ + public interface IPolicyValueBotPolicy : IBotPolicy + { + IPolicyValueNet PolicyValueNet { get; } + } + + public sealed class PolicyValueSearch : ISearch + { + private const double PolicyTargetScoreTemperature = 25000.0; + private readonly IPolicyValueNet _policyValueNet; + + public PolicyValueSearch(IPolicyValueNet policyValueNet) + { + _policyValueNet = policyValueNet; + } + + public string Name => "policy-value-search"; + + public BotTurnDecision Search(Game game, int playerId, IEvaluator evaluator, IBotPolicy opponentPolicy, SearchConfig config) + { + var watch = Stopwatch.StartNew(); + int nodesVisited = 0; + List candidates = PolicyValueTurnLineEnumerator.Enumerate( + game, + playerId, + evaluator, + _policyValueNet, + config.MaxDepth, + config.MaxWidth, + config.CandidateLines, + watch, + config.TimeBudgetMs, + ref nodesVisited); + + if (candidates.Count == 0) + return Fallback(game, watch, nodesVisited); + + TurnLine best = null; + int bestScore = int.MinValue; + var scoredCandidates = new List(); + foreach (TurnLine candidate in candidates) + { + if (watch.ElapsedMilliseconds >= config.TimeBudgetMs) + break; + + int score = ScoreAfterOpponentReply(candidate.Game, playerId, evaluator, opponentPolicy, config.ForOpponent(), watch, config.TimeBudgetMs, ref nodesVisited); + scoredCandidates.Add(new ScoredTurnLine(candidate, score)); + if (best == null || score > bestScore) + { + best = candidate; + bestScore = score; + } + } + + if (best == null) + best = candidates.OrderByDescending(candidate => candidate.Score).First(); + if (scoredCandidates.Count == 0) + scoredCandidates.AddRange(candidates.Select(candidate => new ScoredTurnLine(candidate, candidate.Score))); + + watch.Stop(); + return new BotTurnDecision(best.Tasks, bestScore, watch.Elapsed, Name, nodesVisited, BuildRootActionTargets(scoredCandidates)); + } + + private static int ScoreAfterOpponentReply(Game game, int playerId, IEvaluator evaluator, IBotPolicy opponentPolicy, + SearchConfig opponentConfig, Stopwatch watch, int totalTimeBudgetMs, ref int nodesVisited) + { + if (game.State != State.RUNNING || game.CurrentPlayer.Id == playerId || opponentPolicy == null) + return evaluator.Evaluate(game, playerId).Score; + + int remainingMs = Math.Max(50, totalTimeBudgetMs - (int)watch.ElapsedMilliseconds); + opponentConfig.TimeBudgetMs = Math.Min(opponentConfig.TimeBudgetMs, remainingMs); + IPolicyValueNet opponentNet = (opponentPolicy as IPolicyValueBotPolicy)?.PolicyValueNet; + List replies = PolicyValueTurnLineEnumerator.Enumerate( + game, + game.CurrentPlayer.Id, + opponentPolicy.Evaluator, + opponentNet, + opponentConfig.MaxDepth, + opponentConfig.MaxWidth, + opponentConfig.CandidateLines, + watch, + totalTimeBudgetMs, + ref nodesVisited); + + if (replies.Count == 0) + return evaluator.Evaluate(game, playerId).Score; + + TurnLine opponentBest = replies.OrderByDescending(reply => reply.Score).First(); + return evaluator.Evaluate(opponentBest.Game, playerId).Score; + } + + private static BotTurnDecision Fallback(Game game, Stopwatch watch, int nodesVisited) + { + PlayerTask option = game.CurrentPlayer.Options().FirstOrDefault(task => !(task is ConcedeTask)); + watch.Stop(); + return new BotTurnDecision(option == null ? new List() : new List { option }, 0, watch.Elapsed, "fallback", nodesVisited); + } + + private static IReadOnlyDictionary BuildRootActionTargets(IReadOnlyList scoredLines) + { + var scored = scoredLines + .Where(line => line.Line.Tasks.Count > 0) + .ToList(); + if (scored.Count == 0) + return new Dictionary(); + + double maxScaled = scored.Max(line => line.Score / PolicyTargetScoreTemperature); + var raw = new Dictionary(); + foreach (ScoredTurnLine line in scored) + { + string key = FirebatNaxxPolicyValueNet.ActionKey(line.Line.Tasks[0]); + double value = Math.Exp(Math.Max(-40.0, line.Score / PolicyTargetScoreTemperature - maxScaled)); + if (!raw.ContainsKey(key)) + raw[key] = 0; + raw[key] += value; + } + + double total = raw.Values.Sum(); + if (total <= 0) + return raw.ToDictionary(kvp => kvp.Key, kvp => 1.0 / raw.Count); + return raw.ToDictionary(kvp => kvp.Key, kvp => kvp.Value / total); + } + + private sealed class ScoredTurnLine + { + public ScoredTurnLine(TurnLine line, int score) + { + Line = line; + Score = score; + } + + public TurnLine Line { get; } + public int Score { get; } + } + } + + internal static class PolicyValueTurnLineEnumerator + { + private const int PriorScoreScale = 700; + + public static List Enumerate(Game root, int playerId, IEvaluator evaluator, IPolicyValueNet policyValueNet, + int maxDepth, int maxWidth, int maxLines, Stopwatch watch, int totalTimeBudgetMs, ref int nodesVisited) + { + var frontier = new List + { + new TurnLine(root.Clone(resetRandomSeed: false), new List(), evaluator.Evaluate(root, playerId).Score) + }; + var leaves = new List(); + + for (int depth = 0; depth < maxDepth && frontier.Count > 0; depth++) + { + var nextByHash = new Dictionary(); + foreach (TurnLine line in frontier) + { + if (watch.ElapsedMilliseconds >= totalTimeBudgetMs) + break; + + Game lineGame = line.Game; + if (lineGame.State != State.RUNNING || lineGame.CurrentPlayer.Id != playerId) + { + leaves.Add(line); + continue; + } + + List options = lineGame.CurrentPlayer.Options() + .Where(task => !(task is ConcedeTask)) + .ToList(); + + if (options.Count == 0) + { + leaves.Add(line); + continue; + } + + IReadOnlyDictionary priors = policyValueNet?.Predict(lineGame, playerId, options).ActionPriors + ?? new Dictionary(); + foreach (PlayerTask option in OrderByPrior(options, priors, maxWidth)) + { + if (watch.ElapsedMilliseconds >= totalTimeBudgetMs) + break; + + Game clone = lineGame.Clone(resetRandomSeed: false); + if (!clone.Process(option)) + continue; + + nodesVisited++; + var tasks = new List(line.Tasks) { option }; + double prior = priors.TryGetValue(FirebatNaxxPolicyValueNet.ActionKey(option), out double p) ? p : 0; + int score = evaluator.Evaluate(clone, playerId).Score + (int)(prior * PriorScoreScale); + var next = new TurnLine(clone, tasks, score); + + if (clone.State != State.RUNNING || clone.CurrentPlayer.Id != playerId || option is EndTurnTask) + { + leaves.Add(next); + continue; + } + + string hash = clone.Hash(GameTag.LAST_CARD_PLAYED, GameTag.ENTITY_ID); + if (!nextByHash.TryGetValue(hash, out TurnLine existing) || next.Score > existing.Score) + nextByHash[hash] = next; + } + } + + frontier = nextByHash.Values + .OrderByDescending(line => line.Score) + .Take(maxWidth) + .ToList(); + } + + leaves.AddRange(frontier); + return leaves + .OrderByDescending(line => line.Score) + .Take(maxLines) + .ToList(); + } + + private static IEnumerable OrderByPrior(List options, IReadOnlyDictionary priors, int maxWidth) + { + int limit = Math.Max(maxWidth, 1); + return options + .OrderByDescending(option => priors.TryGetValue(FirebatNaxxPolicyValueNet.ActionKey(option), out double prior) ? prior : 0) + .Take(limit); + } + } +} diff --git a/core-extensions/SabberStoneBasicAI/src/Bots/RandomRolloutDatasetGenerator.cs b/core-extensions/SabberStoneBasicAI/src/Bots/RandomRolloutDatasetGenerator.cs new file mode 100644 index 000000000..d9d41f924 --- /dev/null +++ b/core-extensions/SabberStoneBasicAI/src/Bots/RandomRolloutDatasetGenerator.cs @@ -0,0 +1,540 @@ +#region copyright +// SabberStone, Hearthstone Simulator in C# .NET Core +// Copyright (C) 2017-2019 SabberStone Team, darkfriend77 & rnilva +// +// SabberStone is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License. +// SabberStone is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +#endregion +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using SabberStoneCore.Enums; +using SabberStoneCore.Model; +using SabberStoneCore.Model.Entities; +using SabberStoneCore.Tasks.PlayerTasks; + +namespace SabberStoneBasicAI.Bots +{ + public sealed class RandomRolloutDatasetConfig + { + public int Games { get; set; } = 100; + public long Seed { get; set; } = 1; + public int MaxTurns { get; set; } = 80; + public int MaxActionsPerTurn { get; set; } = 80; + public bool RecordTurnEndSnapshots { get; set; } = true; + public string OutputPath { get; set; } = "firebat-naxx-random-rollouts.jsonl"; + } + + public sealed class RandomRolloutDatasetResult : IRolloutDatasetResult + { + public int Games { get; set; } + public int CompletedGames { get; set; } + public int UnfinishedGames { get; set; } + public int Errors { get; set; } + public int Snapshots { get; set; } + public int LegalValueRows { get; set; } + public int FullInfoValueRows { get; set; } + public int HiddenStateRows { get; set; } + public int PolicyRows { get; set; } + public int RowsWritten => LegalValueRows + FullInfoValueRows + HiddenStateRows + PolicyRows; + public int ActionsPlayed { get; set; } + public int CappedTurns { get; set; } + public string OutputPath { get; set; } + } + + public interface IRolloutDatasetResult + { + int Snapshots { get; set; } + int LegalValueRows { get; set; } + int FullInfoValueRows { get; set; } + int HiddenStateRows { get; set; } + int PolicyRows { get; set; } + } + + public static class RandomRolloutDatasetGenerator + { + private const string Schema = "firebat_naxx_random_rollout_v1"; + private const string PolicyName = "uniform-random-actions-until-end-turn"; + + public static RandomRolloutDatasetResult Generate(RandomRolloutDatasetConfig config) + { + if (config == null) + throw new ArgumentNullException(nameof(config)); + if (config.Games < 1) + throw new ArgumentOutOfRangeException(nameof(config.Games), "Games must be positive."); + if (config.MaxTurns < 1) + throw new ArgumentOutOfRangeException(nameof(config.MaxTurns), "MaxTurns must be positive."); + if (config.MaxActionsPerTurn < 1) + throw new ArgumentOutOfRangeException(nameof(config.MaxActionsPerTurn), "MaxActionsPerTurn must be positive."); + + string outputPath = Path.GetFullPath(config.OutputPath ?? "firebat-naxx-random-rollouts.jsonl"); + string directory = Path.GetDirectoryName(outputPath); + if (!String.IsNullOrEmpty(directory)) + Directory.CreateDirectory(directory); + + var result = new RandomRolloutDatasetResult + { + Games = config.Games, + OutputPath = outputPath + }; + + using (var writer = new StreamWriter(outputPath, false, new UTF8Encoding(false))) + { + for (int i = 0; i < config.Games; i++) + { + long gameSeed = config.Seed + i; + try + { + RunGame(config, gameSeed, writer, result); + } + catch + { + result.Errors++; + } + } + } + + return result; + } + + private static void RunGame(RandomRolloutDatasetConfig config, long seed, TextWriter writer, RandomRolloutDatasetResult result) + { + var snapshots = new List(); + var rng = new Random(unchecked((int)(seed * 1103515245 + 12345))); + Game game = FirebatNaxxGameFactory.Create(seed); + game.StartGame(); + ApplyRandomMulligan(game, game.Player1, rng); + ApplyRandomMulligan(game, game.Player2, rng); + game.MainReady(); + + while (game.State == State.RUNNING && game.Turn <= config.MaxTurns) + { + Controller player = game.CurrentPlayer; + snapshots.Add(CaptureSnapshot(game, seed, player, "turn_start")); + RandomTurnStats turnStats = PlayRandomTurn(game, rng, config.MaxActionsPerTurn, config.RecordTurnEndSnapshots + ? () => + { + if (game.State == State.RUNNING && game.CurrentPlayer == player) + snapshots.Add(CaptureSnapshot(game, seed, player, "turn_end")); + } + : (Action)null); + + result.ActionsPlayed += turnStats.ActionsPlayed; + if (turnStats.Capped) + result.CappedTurns++; + } + + if (game.State == State.COMPLETE) + result.CompletedGames++; + else + result.UnfinishedGames++; + + foreach (RolloutSnapshot snapshot in snapshots) + WriteRows(writer, snapshot, ValueFor(game, snapshot.PlayerId), ResultNameFor(game, snapshot.PlayerId), PolicyName, result); + } + + private static void ApplyRandomMulligan(Game game, Controller controller, Random rng) + { + if (controller.Choice == null) + return; + + List choices = controller.Choice.Choices + .Where(_ => rng.Next(2) == 0) + .ToList(); + game.Process(ChooseTask.Mulligan(controller, choices)); + } + + private static RandomTurnStats PlayRandomTurn(Game game, Random rng, int maxActions, Action captureBeforeEndTurn) + { + int actions = 0; + bool capped = false; + while (game.State == State.RUNNING) + { + List options = game.CurrentPlayer.Options(); + List playable = options + .Where(task => !(task is EndTurnTask) && !(task is ConcedeTask)) + .ToList(); + + if (playable.Count == 0) + { + captureBeforeEndTurn?.Invoke(); + PlayerTask endTurn = options.FirstOrDefault(task => task is EndTurnTask); + if (endTurn != null) + game.Process(endTurn); + break; + } + + if (actions >= maxActions) + { + capped = true; + break; + } + + PlayerTask action = playable[rng.Next(playable.Count)]; + if (!game.Process(action)) + break; + actions++; + } + + return new RandomTurnStats(actions, capped); + } + + internal static RolloutSnapshot CaptureSnapshot(Game game, long seed, Controller viewer, string phase) + { + Controller opponent = viewer.Opponent; + return new RolloutSnapshot + { + GameSeed = seed, + Turn = game.Turn, + Step = game.Step.ToString(), + Phase = phase, + PlayerId = viewer.Id, + Side = SideName(viewer), + LegalObservationJson = SerializeObservation(game, viewer, false), + FullInfoObservationJson = SerializeObservation(game, viewer, true), + HiddenTargetJson = SerializeHiddenTarget(opponent) + }; + } + + internal static void WriteRows(TextWriter writer, RolloutSnapshot snapshot, int value, string resultName, string policyName, IRolloutDatasetResult result) + { + writer.WriteLine(BuildValueRow("legal_value", snapshot, value, resultName, snapshot.LegalObservationJson, policyName)); + result.LegalValueRows++; + writer.WriteLine(BuildValueRow("full_info_value", snapshot, value, resultName, snapshot.FullInfoObservationJson, policyName)); + result.FullInfoValueRows++; + writer.WriteLine(BuildHiddenStateRow(snapshot, policyName)); + result.HiddenStateRows++; + result.Snapshots++; + } + + internal static bool WritePolicyRow(TextWriter writer, RolloutSnapshot snapshot, IReadOnlyList legalActions, PlayerTask selectedAction, string policyName, IRolloutDatasetResult result) + { + return WritePolicyRow(writer, snapshot, legalActions, selectedAction, null, policyName, result); + } + + internal static bool WritePolicyRow(TextWriter writer, RolloutSnapshot snapshot, IReadOnlyList legalActions, PlayerTask selectedAction, + IReadOnlyDictionary actionTargets, string policyName, IRolloutDatasetResult result) + { + if (writer == null || snapshot == null || selectedAction == null || legalActions == null || legalActions.Count == 0) + return false; + + string selectedKey = FirebatNaxxPolicyValueNet.ActionKey(selectedAction); + var legal = legalActions.Where(action => !(action is ConcedeTask)).ToList(); + if (!legal.Any(action => FirebatNaxxPolicyValueNet.ActionKey(action) == selectedKey)) + return false; + + Dictionary targets = BuildPolicyTargets(legal, selectedKey, actionTargets); + if (targets.Count == 0) + return false; + + writer.WriteLine(BuildPolicyRow(snapshot, legal, selectedKey, targets, policyName)); + result.PolicyRows++; + return true; + } + + private static Dictionary BuildPolicyTargets(IReadOnlyList legalActions, string selectedKey, IReadOnlyDictionary actionTargets) + { + var targets = new Dictionary(); + if (actionTargets != null && actionTargets.Count > 0) + { + foreach (PlayerTask action in legalActions) + { + string key = FirebatNaxxPolicyValueNet.ActionKey(action); + if (actionTargets.TryGetValue(key, out double target) && target > 0) + targets[key] = target; + } + } + + if (targets.Count == 0) + targets[selectedKey] = 1.0; + + double total = targets.Values.Sum(); + if (total <= 0) + return new Dictionary(); + return targets.ToDictionary(kvp => kvp.Key, kvp => kvp.Value / total); + } + + private static string BuildValueRow(string kind, RolloutSnapshot snapshot, int value, string resultName, string observationJson, string policyName) + { + var sb = new StringBuilder(); + sb.Append('{'); + AppendCommonFields(sb, kind, snapshot, policyName); + sb.Append(",\"value\":").Append(value.ToString(CultureInfo.InvariantCulture)); + sb.Append(",\"result\":"); + AppendString(sb, resultName); + sb.Append(",\"observation\":").Append(observationJson); + sb.Append('}'); + return sb.ToString(); + } + + private static string BuildHiddenStateRow(RolloutSnapshot snapshot, string policyName) + { + var sb = new StringBuilder(); + sb.Append('{'); + AppendCommonFields(sb, "hidden_state", snapshot, policyName); + sb.Append(",\"observation\":").Append(snapshot.LegalObservationJson); + sb.Append(",\"target\":").Append(snapshot.HiddenTargetJson); + sb.Append('}'); + return sb.ToString(); + } + + private static string BuildPolicyRow(RolloutSnapshot snapshot, IReadOnlyList legalActions, string selectedKey, IReadOnlyDictionary targets, string policyName) + { + var sb = new StringBuilder(); + sb.Append('{'); + AppendCommonFields(sb, "policy", snapshot, policyName); + sb.Append(",\"selectedAction\":"); + AppendString(sb, selectedKey); + sb.Append(",\"legalActions\":["); + for (int i = 0; i < legalActions.Count; i++) + { + PlayerTask action = legalActions[i]; + if (i > 0) + sb.Append(','); + string key = FirebatNaxxPolicyValueNet.ActionKey(action); + sb.Append('{'); + sb.Append("\"key\":"); + AppendString(sb, key); + double target = targets.TryGetValue(key, out double value) ? value : 0; + sb.Append(",\"target\":").Append(target.ToString(CultureInfo.InvariantCulture)); + sb.Append(",\"features\":"); + AppendFeatureObject(sb, FirebatNaxxActionFeatureExtractor.Extract(action.Game, snapshot.PlayerId, action)); + sb.Append('}'); + } + sb.Append(']'); + sb.Append('}'); + return sb.ToString(); + } + + private static void AppendCommonFields(StringBuilder sb, string kind, RolloutSnapshot snapshot, string policyName) + { + sb.Append("\"schema\":"); + AppendString(sb, Schema); + sb.Append(",\"kind\":"); + AppendString(sb, kind); + sb.Append(",\"policy\":"); + AppendString(sb, policyName); + sb.Append(",\"gameSeed\":").Append(snapshot.GameSeed.ToString(CultureInfo.InvariantCulture)); + sb.Append(",\"turn\":").Append(snapshot.Turn.ToString(CultureInfo.InvariantCulture)); + sb.Append(",\"step\":"); + AppendString(sb, snapshot.Step); + sb.Append(",\"phase\":"); + AppendString(sb, snapshot.Phase); + sb.Append(",\"playerId\":").Append(snapshot.PlayerId.ToString(CultureInfo.InvariantCulture)); + sb.Append(",\"side\":"); + AppendString(sb, snapshot.Side); + } + + private static string SerializeObservation(Game game, Controller viewer, bool fullInformation) + { + var sb = new StringBuilder(); + sb.Append('{'); + sb.Append("\"currentPlayerId\":").Append(game.CurrentPlayer.Id.ToString(CultureInfo.InvariantCulture)); + sb.Append(",\"turn\":").Append(game.Turn.ToString(CultureInfo.InvariantCulture)); + sb.Append(",\"step\":"); + AppendString(sb, game.Step.ToString()); + sb.Append(",\"viewer\":"); + AppendControllerObservation(sb, viewer, true, true, fullInformation); + sb.Append(",\"opponent\":"); + AppendControllerObservation(sb, viewer.Opponent, fullInformation, fullInformation, fullInformation); + sb.Append(",\"legalActions\":"); + AppendStringArray(sb, game.State == State.RUNNING && game.CurrentPlayer == viewer + ? viewer.Options().Where(task => !(task is ConcedeTask)).Select(task => task.FullPrint()) + : Enumerable.Empty()); + sb.Append('}'); + return sb.ToString(); + } + + private static void AppendControllerObservation(StringBuilder sb, Controller controller, bool revealHand, bool revealDeck, bool preserveDeckOrder) + { + sb.Append('{'); + sb.Append("\"playerId\":").Append(controller.Id.ToString(CultureInfo.InvariantCulture)); + sb.Append(",\"side\":"); + AppendString(sb, SideName(controller)); + sb.Append(",\"heroHealth\":").Append(controller.Hero.Health.ToString(CultureInfo.InvariantCulture)); + sb.Append(",\"heroArmor\":").Append(controller.Hero.Armor.ToString(CultureInfo.InvariantCulture)); + sb.Append(",\"baseMana\":").Append(controller.BaseMana.ToString(CultureInfo.InvariantCulture)); + sb.Append(",\"remainingMana\":").Append(controller.RemainingMana.ToString(CultureInfo.InvariantCulture)); + sb.Append(",\"handCount\":").Append(controller.HandZone.Count.ToString(CultureInfo.InvariantCulture)); + sb.Append(",\"deckCount\":").Append(controller.DeckZone.Count.ToString(CultureInfo.InvariantCulture)); + sb.Append(",\"board\":"); + AppendBoard(sb, controller); + sb.Append(",\"graveyard\":"); + AppendStringArray(sb, controller.GraveyardZone.Select(card => card.Card.Id)); + + if (revealHand) + { + sb.Append(",\"handIds\":"); + AppendStringArray(sb, controller.HandZone.Select(card => card.Card.Id)); + } + + if (revealDeck) + { + sb.Append(",\"deckIds\":"); + IEnumerable deckIds = controller.DeckZone.Select(card => card.Card.Id); + AppendStringArray(sb, preserveDeckOrder ? deckIds : deckIds.OrderBy(id => id, StringComparer.Ordinal)); + } + + sb.Append('}'); + } + + private static void AppendBoard(StringBuilder sb, Controller controller) + { + sb.Append('['); + bool first = true; + foreach (Minion minion in controller.BoardZone) + { + if (!first) + sb.Append(','); + first = false; + sb.Append('{'); + sb.Append("\"id\":"); + AppendString(sb, minion.Card.Id); + sb.Append(",\"attack\":").Append(minion.AttackDamage.ToString(CultureInfo.InvariantCulture)); + sb.Append(",\"health\":").Append(minion.Health.ToString(CultureInfo.InvariantCulture)); + sb.Append(",\"taunt\":").Append(minion.HasTaunt ? "true" : "false"); + sb.Append(",\"charge\":").Append(minion.HasCharge ? "true" : "false"); + sb.Append(",\"stealth\":").Append(minion.HasStealth ? "true" : "false"); + sb.Append('}'); + } + sb.Append(']'); + } + + private static string SerializeHiddenTarget(Controller hidden) + { + var sb = new StringBuilder(); + sb.Append('{'); + sb.Append("\"hiddenPlayerId\":").Append(hidden.Id.ToString(CultureInfo.InvariantCulture)); + sb.Append(",\"handIds\":"); + AppendStringArray(sb, hidden.HandZone.Select(card => card.Card.Id)); + sb.Append(",\"deckIds\":"); + AppendStringArray(sb, hidden.DeckZone.Select(card => card.Card.Id)); + sb.Append('}'); + return sb.ToString(); + } + + internal static int ValueFor(Game game, int playerId) + { + Controller player = game.ControllerById(playerId); + if (player.PlayState == PlayState.WON) + return 1; + if (player.PlayState == PlayState.LOST || player.PlayState == PlayState.CONCEDED) + return -1; + return 0; + } + + internal static string ResultNameFor(Game game, int playerId) + { + return game.ControllerById(playerId).PlayState.ToString(); + } + + private static string SideName(Controller controller) + { + return controller.HeroClass == CardClass.DRUID ? "combo_druid" : "zoo_warlock"; + } + + private static void AppendStringArray(StringBuilder sb, IEnumerable values) + { + sb.Append('['); + bool first = true; + foreach (string value in values) + { + if (!first) + sb.Append(','); + first = false; + AppendString(sb, value); + } + sb.Append(']'); + } + + private static void AppendFeatureObject(StringBuilder sb, IReadOnlyDictionary features) + { + sb.Append('{'); + bool first = true; + foreach (KeyValuePair feature in features + .Where(kvp => kvp.Value != 0) + .OrderBy(kvp => kvp.Key, StringComparer.Ordinal)) + { + if (!first) + sb.Append(','); + first = false; + AppendString(sb, feature.Key); + sb.Append(':').Append(feature.Value.ToString(CultureInfo.InvariantCulture)); + } + sb.Append('}'); + } + + private static void AppendString(StringBuilder sb, string value) + { + if (value == null) + { + sb.Append("null"); + return; + } + + sb.Append('"'); + foreach (char c in value) + { + switch (c) + { + case '\\': + sb.Append("\\\\"); + break; + case '"': + sb.Append("\\\""); + break; + case '\n': + sb.Append("\\n"); + break; + case '\r': + sb.Append("\\r"); + break; + case '\t': + sb.Append("\\t"); + break; + default: + if (c < 32) + sb.Append("\\u").Append(((int)c).ToString("x4", CultureInfo.InvariantCulture)); + else + sb.Append(c); + break; + } + } + sb.Append('"'); + } + + internal sealed class RolloutSnapshot + { + public long GameSeed { get; set; } + public int Turn { get; set; } + public string Step { get; set; } + public string Phase { get; set; } + public int PlayerId { get; set; } + public string Side { get; set; } + public string LegalObservationJson { get; set; } + public string FullInfoObservationJson { get; set; } + public string HiddenTargetJson { get; set; } + } + + private sealed class RandomTurnStats + { + public RandomTurnStats(int actionsPlayed, bool capped) + { + ActionsPlayed = actionsPlayed; + Capped = capped; + } + + public int ActionsPlayed { get; } + public bool Capped { get; } + } + } +} diff --git a/core-extensions/SabberStoneBasicAI/src/Bots/SelfPlayRolloutDatasetGenerator.cs b/core-extensions/SabberStoneBasicAI/src/Bots/SelfPlayRolloutDatasetGenerator.cs new file mode 100644 index 000000000..d2c0c7b5f --- /dev/null +++ b/core-extensions/SabberStoneBasicAI/src/Bots/SelfPlayRolloutDatasetGenerator.cs @@ -0,0 +1,234 @@ +#region copyright +// SabberStone, Hearthstone Simulator in C# .NET Core +// Copyright (C) 2017-2019 SabberStone Team, darkfriend77 & rnilva +// +// SabberStone is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License. +// SabberStone is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +#endregion +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using SabberStoneCore.Enums; +using SabberStoneCore.Model; +using SabberStoneCore.Model.Entities; +using SabberStoneCore.Tasks.PlayerTasks; + +namespace SabberStoneBasicAI.Bots +{ + public sealed class SelfPlayRolloutDatasetConfig + { + public int Games { get; set; } = 100; + public long Seed { get; set; } = 1; + public int MaxTurns { get; set; } = 80; + public int MaxDecisions { get; set; } = 1000; + public bool RecordTurnEndSnapshots { get; set; } = true; + public string OutputPath { get; set; } = "firebat-naxx-selfplay-rollouts.jsonl"; + public string Policy { get; set; } = "baseline"; + public string WeightsPath { get; set; } + public SearchConfig SearchConfig { get; set; } = new SearchConfig(); + public int ProgressIntervalGames { get; set; } + public Action Progress { get; set; } + } + + public sealed class SelfPlayRolloutProgress + { + public int Games { get; set; } + public int GamesProcessed { get; set; } + public long LastGameSeed { get; set; } + public TimeSpan Elapsed { get; set; } + public SelfPlayRolloutDatasetResult Result { get; set; } + } + + public sealed class SelfPlayRolloutDatasetResult : IRolloutDatasetResult + { + public int Games { get; set; } + public int CompletedGames { get; set; } + public int UnfinishedGames { get; set; } + public int Errors { get; set; } + public int Snapshots { get; set; } + public int LegalValueRows { get; set; } + public int FullInfoValueRows { get; set; } + public int HiddenStateRows { get; set; } + public int PolicyRows { get; set; } + public int RowsWritten => LegalValueRows + FullInfoValueRows + HiddenStateRows + PolicyRows; + public int ActionsPlayed { get; set; } + public int Decisions { get; set; } + public int DecisionCappedGames { get; set; } + public int NodesVisited { get; set; } + public TimeSpan SearchElapsed { get; set; } + public string OutputPath { get; set; } + public string PolicyName { get; set; } + public double AverageMoveMs => Decisions == 0 ? 0 : SearchElapsed.TotalMilliseconds / Decisions; + } + + public static class SelfPlayRolloutDatasetGenerator + { + public static SelfPlayRolloutDatasetResult Generate(SelfPlayRolloutDatasetConfig config) + { + if (config == null) + throw new ArgumentNullException(nameof(config)); + if (config.Games < 1) + throw new ArgumentOutOfRangeException(nameof(config.Games), "Games must be positive."); + if (config.MaxTurns < 1) + throw new ArgumentOutOfRangeException(nameof(config.MaxTurns), "MaxTurns must be positive."); + if (config.MaxDecisions < 1) + throw new ArgumentOutOfRangeException(nameof(config.MaxDecisions), "MaxDecisions must be positive."); + if (config.SearchConfig == null) + config.SearchConfig = new SearchConfig(); + + FirebatNaxxPolicySet policies = FirebatNaxxPolicyFactory.Create(config.Policy, config.WeightsPath, "selfplay", config.Seed); + string outputPath = Path.GetFullPath(config.OutputPath ?? "firebat-naxx-selfplay-rollouts.jsonl"); + string directory = Path.GetDirectoryName(outputPath); + if (!String.IsNullOrEmpty(directory)) + Directory.CreateDirectory(directory); + + var result = new SelfPlayRolloutDatasetResult + { + Games = config.Games, + OutputPath = outputPath, + PolicyName = policies.Name + }; + + DateTime started = DateTime.UtcNow; + using (var writer = new StreamWriter(outputPath, false, new UTF8Encoding(false))) + { + for (int i = 0; i < config.Games; i++) + { + long gameSeed = config.Seed + i; + try + { + RunGame(config, policies, gameSeed, writer, result); + } + catch + { + result.Errors++; + } + + if (ShouldReportProgress(config, i + 1)) + { + writer.Flush(); + config.Progress(new SelfPlayRolloutProgress + { + Games = config.Games, + GamesProcessed = i + 1, + LastGameSeed = gameSeed, + Elapsed = DateTime.UtcNow - started, + Result = result + }); + } + } + } + + return result; + } + + private static bool ShouldReportProgress(SelfPlayRolloutDatasetConfig config, int gamesProcessed) + { + if (config.Progress == null) + return false; + int interval = Math.Max(1, config.ProgressIntervalGames); + return gamesProcessed == config.Games || gamesProcessed % interval == 0; + } + + private static void RunGame(SelfPlayRolloutDatasetConfig config, FirebatNaxxPolicySet policies, long seed, TextWriter writer, SelfPlayRolloutDatasetResult result) + { + var snapshots = new List(); + Game game = FirebatNaxxGameFactory.Create(seed); + game.StartGame(); + ApplyMulligan(game, game.Player1, policies.Druid); + ApplyMulligan(game, game.Player2, policies.Zoo); + game.MainReady(); + + int decisions = 0; + while (game.State == State.RUNNING && game.Turn <= config.MaxTurns && decisions < config.MaxDecisions) + { + Controller player = game.CurrentPlayer; + IBotPolicy currentPolicy = player == game.Player1 ? policies.Druid : policies.Zoo; + IBotPolicy opponentPolicy = player == game.Player1 ? policies.Zoo : policies.Druid; + int currentPlayerId = player.Id; + + RandomRolloutDatasetGenerator.RolloutSnapshot snapshot = RandomRolloutDatasetGenerator.CaptureSnapshot(game, seed, player, "turn_start"); + snapshots.Add(snapshot); + List legalActions = player.Options().Where(task => !(task is ConcedeTask)).ToList(); + BotTurnDecision decision = currentPolicy.SelectTurn(game, player, new BotTurnContext(config.SearchConfig, opponentPolicy)); + decisions++; + result.NodesVisited += decision.NodesVisited; + result.SearchElapsed += decision.Elapsed; + RandomRolloutDatasetGenerator.WritePolicyRow(writer, snapshot, legalActions, SelectedRootAction(decision, legalActions), decision.RootActionTargets, policies.Name, result); + + bool processed = ProcessDecision(game, currentPlayerId, decision, config.RecordTurnEndSnapshots + ? () => + { + if (game.State == State.RUNNING && game.CurrentPlayer == player) + snapshots.Add(RandomRolloutDatasetGenerator.CaptureSnapshot(game, seed, player, "turn_end")); + } + : (Action)null, result); + + if (!processed) + break; + } + + if (game.State == State.RUNNING && decisions >= config.MaxDecisions) + { + result.DecisionCappedGames++; + } + + result.Decisions += decisions; + if (game.State == State.COMPLETE) + result.CompletedGames++; + else + result.UnfinishedGames++; + + foreach (RandomRolloutDatasetGenerator.RolloutSnapshot snapshot in snapshots) + RandomRolloutDatasetGenerator.WriteRows(writer, snapshot, RandomRolloutDatasetGenerator.ValueFor(game, snapshot.PlayerId), + RandomRolloutDatasetGenerator.ResultNameFor(game, snapshot.PlayerId), policies.Name, result); + } + + private static void ApplyMulligan(Game game, Controller controller, IBotPolicy policy) + { + if (controller.Choice == null) + return; + + game.Process(ChooseTask.Mulligan(controller, policy.Mulligan(game, controller))); + } + + private static bool ProcessDecision(Game game, int playerId, BotTurnDecision decision, Action captureBeforeEndTurn, SelfPlayRolloutDatasetResult result) + { + IReadOnlyList tasks = decision.Tasks; + if (tasks.Count == 0) + tasks = game.CurrentPlayer.Options().Where(task => !(task is ConcedeTask)).Take(1).ToList(); + + foreach (PlayerTask task in tasks) + { + if (game.State != State.RUNNING || game.CurrentPlayer.Id != playerId) + break; + + if (task is EndTurnTask) + captureBeforeEndTurn?.Invoke(); + + if (!game.Process(task)) + return false; + + result.ActionsPlayed++; + } + + return true; + } + + private static PlayerTask SelectedRootAction(BotTurnDecision decision, IReadOnlyList legalActions) + { + if (decision.Tasks.Count > 0) + return decision.Tasks[0]; + return legalActions.FirstOrDefault(); + } + + } +} diff --git a/core-extensions/SabberStoneBasicAI/src/FirebatNaxxMatchupRunner.cs b/core-extensions/SabberStoneBasicAI/src/FirebatNaxxMatchupRunner.cs new file mode 100644 index 000000000..40273e7e3 --- /dev/null +++ b/core-extensions/SabberStoneBasicAI/src/FirebatNaxxMatchupRunner.cs @@ -0,0 +1,175 @@ +#region copyright +// SabberStone, Hearthstone Simulator in C# .NET Core +// Copyright (C) 2017-2019 SabberStone Team, darkfriend77 & rnilva +// +// SabberStone is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License. +// SabberStone is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +#endregion +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using SabberStoneBasicAI.Meta; +using SabberStoneBasicAI.Nodes; +using SabberStoneBasicAI.Score; +using SabberStoneCore.Config; +using SabberStoneCore.Enums; +using SabberStoneCore.Model; +using SabberStoneCore.Model.Entities; +using SabberStoneCore.Tasks.PlayerTasks; + +namespace SabberStoneBasicAI +{ + public static class FirebatNaxxMatchupRunner + { + private const string DruidName = "Firebat Combo Druid"; + private const string WarlockName = "Firebat Zoo Warlock"; + + public static void Run(int games = 10, int maxDepth = 8, int maxWidth = 80, int maxTurns = 80, + long? seed = 1, bool verbose = false) + { + ValidateDeck("Combo Druid", Decks.FirebatNaxxComboDruid); + ValidateDeck("Zoo Warlock", Decks.FirebatNaxxZooWarlock); + + int druidWins = 0; + int warlockWins = 0; + int ties = 0; + int unfinished = 0; + int totalTurns = 0; + var watch = Stopwatch.StartNew(); + + for (int i = 0; i < games; i++) + { + var game = CreateGame(seed.HasValue ? seed + i : null); + var druidScore = new ComboDruidScore(); + var warlockScore = new ZooWarlockScore(); + + game.StartGame(); + ApplyMulligan(game, game.Player1, druidScore); + ApplyMulligan(game, game.Player2, warlockScore); + game.MainReady(); + + int actions = 0; + while (game.State != State.COMPLETE && game.Turn <= maxTurns && actions < 1000) + { + IScore score = game.CurrentPlayer == game.Player1 ? druidScore : warlockScore; + if (!ProcessBestLine(game, score, maxDepth, maxWidth, verbose)) + break; + actions++; + } + + totalTurns += game.Turn; + + if (game.State != State.COMPLETE) + unfinished++; + else if (game.Player1.PlayState == PlayState.WON) + druidWins++; + else if (game.Player2.PlayState == PlayState.WON) + warlockWins++; + else + ties++; + + Console.WriteLine( + $"{i + 1,3}. {DruidName}: {game.Player1.PlayState}, {WarlockName}: {game.Player2.PlayState}, turns={game.Turn}, actions={actions}"); + } + + watch.Stop(); + + Console.WriteLine(); + Console.WriteLine($"Games: {games}"); + Console.WriteLine($"{DruidName} wins: {druidWins}"); + Console.WriteLine($"{WarlockName} wins: {warlockWins}"); + Console.WriteLine($"Ties: {ties}"); + Console.WriteLine($"Unfinished: {unfinished}"); + Console.WriteLine($"Average turns: {(games == 0 ? 0 : totalTurns / (double)games):0.0}"); + Console.WriteLine($"Elapsed: {watch.Elapsed}"); + } + + private static Game CreateGame(long? seed) + { + return new Game(new GameConfig + { + StartPlayer = -1, + Player1Name = DruidName, + Player1HeroClass = CardClass.DRUID, + Player1Deck = Decks.FirebatNaxxComboDruid, + Player2Name = WarlockName, + Player2HeroClass = CardClass.WARLOCK, + Player2Deck = Decks.FirebatNaxxZooWarlock, + FormatType = FormatType.FT_WILD, + FillDecks = false, + Shuffle = true, + SkipMulligan = false, + History = false, + Logging = false, + RandomSeed = seed + }); + } + + private static void ApplyMulligan(Game game, Controller controller, IScore score) + { + if (controller.Choice == null) + return; + + List choices = controller.Choice.Choices + .Select(id => game.IdEntityDic[id]) + .ToList(); + List mulligan = score.MulliganRule().Invoke(choices); + game.Process(ChooseTask.Mulligan(controller, mulligan)); + } + + private static bool ProcessBestLine(Game game, IScore score, int maxDepth, int maxWidth, bool verbose) + { + Controller player = game.CurrentPlayer; + int playerId = player.Id; + List solutions = OptionNode.GetSolutions(game, playerId, score, maxDepth, maxWidth); + + if (solutions.Count == 0) + return ProcessFallbackOption(game, verbose); + + var tasks = new List(); + solutions.OrderByDescending(solution => solution.Score).First().PlayerTasks(ref tasks); + + if (tasks.Count == 0) + return ProcessFallbackOption(game, verbose); + + foreach (PlayerTask task in tasks) + { + if (verbose) + Console.WriteLine(task.FullPrint()); + + game.Process(task); + + if (game.State != State.RUNNING || game.CurrentPlayer.Id != playerId || game.CurrentPlayer.Choice != null) + break; + } + + return true; + } + + private static bool ProcessFallbackOption(Game game, bool verbose) + { + PlayerTask option = game.CurrentPlayer.Options().FirstOrDefault(); + if (option == null) + return false; + + if (verbose) + Console.WriteLine(option.FullPrint()); + + game.Process(option); + return true; + } + + private static void ValidateDeck(string name, List deck) + { + if (deck.Count != 30) + throw new InvalidOperationException($"{name} has {deck.Count} cards; expected 30."); + } + } +} diff --git a/core-extensions/SabberStoneBasicAI/src/Meta/DeckTypes.cs b/core-extensions/SabberStoneBasicAI/src/Meta/DeckTypes.cs index 45051df92..59c9eda18 100644 --- a/core-extensions/SabberStoneBasicAI/src/Meta/DeckTypes.cs +++ b/core-extensions/SabberStoneBasicAI/src/Meta/DeckTypes.cs @@ -15,6 +15,8 @@ namespace SabberStoneBasicAI.Meta { public enum DeckTypes { + FirebatNaxxComboDruid, + FirebatNaxxZooWarlock, AggroPirateWarrior, MurlocDruid, MidrangeJadeShaman, @@ -25,4 +27,4 @@ public enum DeckTypes MiraclePirateRogue, RenoKazakusDragonPriest } -} \ No newline at end of file +} diff --git a/core-extensions/SabberStoneBasicAI/src/Meta/Decks.cs b/core-extensions/SabberStoneBasicAI/src/Meta/Decks.cs index 683c0291a..edba8d6fb 100644 --- a/core-extensions/SabberStoneBasicAI/src/Meta/Decks.cs +++ b/core-extensions/SabberStoneBasicAI/src/Meta/Decks.cs @@ -18,6 +18,80 @@ namespace SabberStoneBasicAI.Meta { public class Decks { + /// + /// Firebat's Combo Druid from the 2014 World Championship/Naxxramas-era lineup. + /// + public static List FirebatNaxxComboDruid => new List() + { + Cards.FromId("FB_Champs_EX1_169"), + Cards.FromId("FB_Champs_EX1_169"), + Cards.FromId("CS2_013"), + Cards.FromId("CS2_013"), + Cards.FromId("EX1_154"), + Cards.FromId("EX1_154"), + Cards.FromId("CS2_011"), + Cards.FromId("CS2_011"), + Cards.FromId("CS2_012"), + Cards.FromId("CS2_012"), + Cards.FromId("FB_Champs_EX1_166"), + Cards.FromId("FB_Champs_EX1_166"), + Cards.FromId("EX1_571"), + Cards.FromId("FB_Champs_EX1_165"), + Cards.FromId("FB_Champs_EX1_165"), + Cards.FromId("FB_Champs_NEW1_008"), + Cards.FromId("FB_Champs_NEW1_008"), + Cards.FromId("EX1_573"), + Cards.FromId("EX1_058"), + Cards.FromId("FP1_005"), + Cards.FromId("FP1_005"), + Cards.FromId("FB_Champs_EX1_005"), + Cards.FromId("CS2_182"), + Cards.FromId("CS2_182"), + Cards.FromId("EX1_002"), + Cards.FromId("EX1_284"), + Cards.FromId("FP1_030"), + Cards.FromId("FP1_008"), + Cards.FromId("FP1_008"), + Cards.FromId("EX1_110") + }; + + /// + /// Firebat's Zoo Warlock from the 2014 World Championship/Naxxramas-era lineup. + /// + public static List FirebatNaxxZooWarlock => new List() + { + Cards.FromId("EX1_319"), + Cards.FromId("EX1_319"), + Cards.FromId("EX1_316"), + Cards.FromId("EX1_308"), + Cards.FromId("EX1_308"), + Cards.FromId("CS2_065"), + Cards.FromId("CS2_065"), + Cards.FromId("EX1_304"), + Cards.FromId("EX1_310"), + Cards.FromId("EX1_310"), + Cards.FromId("FB_Champs_CS2_188"), + Cards.FromId("FB_Champs_CS2_188"), + Cards.FromId("FB_Champs_EX1_029"), + Cards.FromId("FB_Champs_EX1_029"), + Cards.FromId("FB_Champs_FP1_028"), + Cards.FromId("FB_Champs_FP1_028"), + Cards.FromId("EX1_162"), + Cards.FromId("EX1_162"), + Cards.FromId("FP1_002"), + Cards.FromId("FP1_002"), + Cards.FromId("FB_Champs_NEW1_019"), + Cards.FromId("FB_Champs_NEW1_019"), + Cards.FromId("FP1_007"), + Cards.FromId("FP1_007"), + Cards.FromId("FB_Champs_EX1_556"), + Cards.FromId("FB_Champs_EX1_556"), + Cards.FromId("EX1_046"), + Cards.FromId("EX1_093"), + Cards.FromId("EX1_093"), + Cards.FromId("FP1_030") + }; + /// /// Questing Miracle Rogue Deck List Guide (January 2017, Standard) – Season 34 /// http://www.hearthstonetopdecks.com/decks/miracle-pirate-rogue-deck-list-guide-standard/ diff --git a/core-extensions/SabberStoneBasicAI/src/Program.cs b/core-extensions/SabberStoneBasicAI/src/Program.cs index 426b61f58..bd955f73e 100644 --- a/core-extensions/SabberStoneBasicAI/src/Program.cs +++ b/core-extensions/SabberStoneBasicAI/src/Program.cs @@ -14,11 +14,14 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.IO; using System.Linq; +using System.Text; using SabberStoneCore.Config; using SabberStoneCore.Enums; using SabberStoneCore.Model; using SabberStoneCore.Tasks.PlayerTasks; +using SabberStoneBasicAI.Bots; using SabberStoneBasicAI.Meta; using SabberStoneBasicAI.Nodes; using SabberStoneBasicAI.Score; @@ -29,19 +32,487 @@ internal class Program { private static readonly Random Rnd = new Random(); - private static void Main() + private static void Main(string[] args) { - Console.WriteLine("Starting test setup."); + if (args.Length > 0 && args[0].Equals("benchmark", StringComparison.OrdinalIgnoreCase)) + { + RunBenchmarkCommand(args.Skip(1).ToArray()); + return; + } + + if (args.Length > 0 && args[0].Equals("pv-benchmark", StringComparison.OrdinalIgnoreCase)) + { + RunPolicyValueBenchmarkCommand(args.Skip(1).ToArray()); + return; + } + + if (args.Length > 0 && args[0].Equals("pv-arena", StringComparison.OrdinalIgnoreCase)) + { + RunPolicyValueArenaCommand(args.Skip(1).ToArray()); + return; + } + + if (args.Length > 0 && args[0].Equals("tune", StringComparison.OrdinalIgnoreCase)) + { + RunTuneCommand(args.Skip(1).ToArray()); + return; + } + + if (args.Length > 0 && args[0].Equals("generate-random-rollouts", StringComparison.OrdinalIgnoreCase)) + { + RunGenerateRandomRolloutsCommand(args.Skip(1).ToArray()); + return; + } + + if (args.Length > 0 && args[0].Equals("generate-selfplay-rollouts", StringComparison.OrdinalIgnoreCase)) + { + RunGenerateSelfPlayRolloutsCommand(args.Skip(1).ToArray()); + return; + } + + if (args.Length > 0 && args[0].Equals("self-play-iteration", StringComparison.OrdinalIgnoreCase)) + { + RunSelfPlayIterationCommand(args.Skip(1).ToArray()); + return; + } + + if (args.Length > 0 && args[0].Equals("show-game", StringComparison.OrdinalIgnoreCase)) + { + RunShowGameCommand(args.Skip(1).ToArray()); + return; + } + + if (args.Length > 0 && args[0].Equals("play", StringComparison.OrdinalIgnoreCase)) + { + RunPlayCommand(args.Skip(1).ToArray()); + return; + } + + if (args.Length > 0 && args[0].Equals("pv-play", StringComparison.OrdinalIgnoreCase)) + { + RunPolicyValuePlayCommand(args.Skip(1).ToArray()); + return; + } + + RunLegacyMatchupCommand(args); + } + + private static int ReadInt(string[] args, int index, int fallback) + { + return args.Length > index && int.TryParse(args[index], out int value) ? value : fallback; + } + + private static void RunLegacyMatchupCommand(string[] args) + { + int games = ReadInt(args, 0, 10); + int maxDepth = ReadInt(args, 1, 8); + int maxWidth = ReadInt(args, 2, 80); + bool verbose = HasFlag(args, "--verbose"); + + Console.WriteLine("Firebat Naxx matchup: Combo Druid vs Zoo Warlock"); + Console.WriteLine($"games={games}, maxDepth={maxDepth}, maxWidth={maxWidth}"); + Console.WriteLine(); + + FirebatNaxxMatchupRunner.Run(games, maxDepth, maxWidth, verbose: verbose); + } + + private static void RunBenchmarkCommand(string[] args) + { + int games = ReadOption(args, "--games", 20); + long seed = ReadLongOption(args, "--seed", 1); + int maxTurns = ReadOption(args, "--max-turns", 80); + SearchConfig config = ReadSearchConfig(args); + + Console.WriteLine("Firebat Naxx benchmark"); + Console.WriteLine($"games={games}, seed={seed}, {SearchConfigSummary(config)}"); + FirebatNaxxBenchmarkRunner.PrintReport(FirebatNaxxBenchmarkRunner.RunBenchmark(games, seed, config, maxTurns, HasFlag(args, "--verbose"))); + } + + private static void RunPolicyValueBenchmarkCommand(string[] args) + { + int games = ReadOption(args, "--games", 20); + long seed = ReadLongOption(args, "--seed", 1); + int maxTurns = ReadOption(args, "--max-turns", 80); + SearchConfig config = ReadSearchConfig(args); + string weights = ReadStringOption(args, "--weights", null); + + Console.WriteLine(weights == null ? "Firebat Naxx policy-value benchmark" : "Firebat Naxx trained policy-value benchmark"); + Console.WriteLine($"games={games}, seed={seed}, {SearchConfigSummary(config)}"); + if (weights != null) + Console.WriteLine($"weights={weights}"); + FirebatNaxxBenchmarkRunner.PrintReport(weights == null + ? FirebatNaxxBenchmarkRunner.RunPolicyValueBenchmark(games, seed, config, maxTurns, HasFlag(args, "--verbose")) + : FirebatNaxxBenchmarkRunner.RunTrainedPolicyValueBenchmark(weights, games, seed, config, maxTurns, HasFlag(args, "--verbose"))); + } + + private static void RunPolicyValueArenaCommand(string[] args) + { + int games = ReadOption(args, "--games", 20); + long seed = ReadLongOption(args, "--seed", 1); + int maxTurns = ReadOption(args, "--max-turns", 80); + SearchConfig config = ReadSearchConfig(args); + string candidate = ReadStringOption(args, "--candidate", null); + string incumbent = ReadStringOption(args, "--incumbent", null); + if (String.IsNullOrWhiteSpace(candidate) || String.IsNullOrWhiteSpace(incumbent)) + throw new ArgumentException("pv-arena requires --candidate and --incumbent weights paths."); + + Console.WriteLine("Firebat Naxx trained policy-value arena"); + Console.WriteLine($"games={games}, seed={seed}, candidate={candidate}, incumbent={incumbent}"); + Console.WriteLine(SearchConfigSummary(config)); + FirebatNaxxBenchmarkReport report = FirebatNaxxBenchmarkRunner.RunTrainedPolicyValueArena(candidate, incumbent, games, seed, config, maxTurns, HasFlag(args, "--verbose")); + FirebatNaxxBenchmarkRunner.PrintReport(report); + PrintArenaSummary(report, ReadDoubleOption(args, "--promotion-threshold", 0.55)); + } + + private static void RunTuneCommand(string[] args) + { + int games = ReadOption(args, "--games", 10); + long seed = ReadLongOption(args, "--seed", 1); + int maxTurns = ReadOption(args, "--max-turns", 80); + SearchConfig config = ReadSearchConfig(args); + + Console.WriteLine("Firebat Naxx tuning"); + Console.WriteLine($"gamesPerPreset={games}, seed={seed}"); + Console.WriteLine(SearchConfigSummary(config)); + FirebatNaxxBenchmarkRunner.RunTuning(games, seed, config, maxTurns); + } + + private static void RunPlayCommand(string[] args) + { + int games = ReadOption(args, "--games", ReadInt(args, 0, 10)); + long seed = ReadLongOption(args, "--seed", 1); + int maxTurns = ReadOption(args, "--max-turns", 80); + SearchConfig config = ReadSearchConfig(args); + + Console.WriteLine("Firebat Naxx candidate mirror"); + Console.WriteLine(SearchConfigSummary(config)); + FirebatNaxxBenchmarkRunner.PrintReport(new FirebatNaxxBenchmarkReport(new[] + { + FirebatNaxxBenchmarkRunner.RunCandidateMirror(games, seed, config, maxTurns, HasFlag(args, "--verbose")) + })); + } + + private static void RunPolicyValuePlayCommand(string[] args) + { + int games = ReadOption(args, "--games", ReadInt(args, 0, 10)); + long seed = ReadLongOption(args, "--seed", 1); + int maxTurns = ReadOption(args, "--max-turns", 80); + SearchConfig config = ReadSearchConfig(args); + string weights = ReadStringOption(args, "--weights", null); + + Console.WriteLine(weights == null ? "Firebat Naxx policy-value mirror" : "Firebat Naxx trained policy-value mirror"); + Console.WriteLine(SearchConfigSummary(config)); + if (weights != null) + Console.WriteLine($"weights={weights}"); + FirebatNaxxBenchmarkRunner.PrintReport(new FirebatNaxxBenchmarkReport(new[] + { + weights == null + ? FirebatNaxxBenchmarkRunner.RunPolicyValueMirror(games, seed, config, maxTurns, HasFlag(args, "--verbose")) + : FirebatNaxxBenchmarkRunner.RunTrainedPolicyValueMirror(weights, games, seed, config, maxTurns, HasFlag(args, "--verbose")) + })); + } + + private static void RunGenerateRandomRolloutsCommand(string[] args) + { + var config = new RandomRolloutDatasetConfig + { + Games = ReadOption(args, "--games", 100), + Seed = ReadLongOption(args, "--seed", 1), + MaxTurns = ReadOption(args, "--max-turns", 80), + MaxActionsPerTurn = ReadOption(args, "--max-actions-per-turn", 80), + OutputPath = ReadStringOption(args, "--output", "firebat-naxx-random-rollouts.jsonl"), + RecordTurnEndSnapshots = !HasFlag(args, "--turn-start-only") + }; + + Console.WriteLine("Firebat Naxx random rollout dataset"); + Console.WriteLine($"games={config.Games}, seed={config.Seed}, maxTurns={config.MaxTurns}, maxActionsPerTurn={config.MaxActionsPerTurn}"); + Console.WriteLine($"output={config.OutputPath}"); + + RandomRolloutDatasetResult result = RandomRolloutDatasetGenerator.Generate(config); + Console.WriteLine($"completed={result.CompletedGames}, unfinished={result.UnfinishedGames}, errors={result.Errors}, cappedTurns={result.CappedTurns}"); + Console.WriteLine($"snapshots={result.Snapshots}, rows={result.RowsWritten}, legalValueRows={result.LegalValueRows}, fullInfoValueRows={result.FullInfoValueRows}, hiddenStateRows={result.HiddenStateRows}, policyRows={result.PolicyRows}"); + Console.WriteLine($"wrote {result.OutputPath}"); + } + + private static void RunGenerateSelfPlayRolloutsCommand(string[] args) + { + var config = new SelfPlayRolloutDatasetConfig + { + Games = ReadOption(args, "--games", 100), + Seed = ReadLongOption(args, "--seed", 1), + MaxTurns = ReadOption(args, "--max-turns", 80), + MaxDecisions = ReadOption(args, "--max-decisions", 1000), + OutputPath = ReadStringOption(args, "--output", "firebat-naxx-selfplay-rollouts.jsonl"), + RecordTurnEndSnapshots = !HasFlag(args, "--turn-start-only"), + Policy = ReadStringOption(args, "--policy", "baseline"), + WeightsPath = ReadStringOption(args, "--weights", null), + SearchConfig = ReadSearchConfig(args) + }; + if (HasFlag(args, "--progress")) + { + config.ProgressIntervalGames = ReadOption(args, "--progress-every", 1); + config.Progress = PrintSelfPlayProgress; + } + + Console.WriteLine("Firebat Naxx self-play rollout dataset"); + Console.WriteLine($"games={config.Games}, seed={config.Seed}, policy={config.Policy}, maxTurns={config.MaxTurns}, maxDecisions={config.MaxDecisions}"); + Console.WriteLine(SearchConfigSummary(config.SearchConfig)); + if (config.WeightsPath != null) + Console.WriteLine($"weights={config.WeightsPath}"); + Console.WriteLine($"output={config.OutputPath}"); + + SelfPlayRolloutDatasetResult result = SelfPlayRolloutDatasetGenerator.Generate(config); + Console.WriteLine($"completed={result.CompletedGames}, unfinished={result.UnfinishedGames}, errors={result.Errors}, decisionCappedGames={result.DecisionCappedGames}"); + Console.WriteLine($"decisions={result.Decisions}, actions={result.ActionsPlayed}, avgMoveMs={result.AverageMoveMs:0.0}, nodes={result.NodesVisited}"); + Console.WriteLine($"snapshots={result.Snapshots}, rows={result.RowsWritten}, legalValueRows={result.LegalValueRows}, fullInfoValueRows={result.FullInfoValueRows}, hiddenStateRows={result.HiddenStateRows}, policyRows={result.PolicyRows}"); + Console.WriteLine($"policy={result.PolicyName}"); + Console.WriteLine($"wrote {result.OutputPath}"); + } + + private static void RunSelfPlayIterationCommand(string[] args) + { + int iteration = ReadOption(args, "--iteration", 1); + string root = Path.GetFullPath(ReadStringOption(args, "--output-dir", Path.Combine("data", "iterations"))); + string iterationName = $"iter-{Math.Max(1, iteration):000}"; + string iterationDir = Path.Combine(root, iterationName); + string championPath = Path.Combine(root, "champion.json"); + Directory.CreateDirectory(iterationDir); + + string inputWeights = ReadStringOption(args, "--weights", null); + if (String.IsNullOrWhiteSpace(inputWeights) && File.Exists(championPath)) + { + inputWeights = championPath; + } + else if (String.IsNullOrWhiteSpace(inputWeights) && iteration > 1) + { + string previousWeights = Path.Combine(root, $"iter-{iteration - 1:000}", "weights.json"); + if (File.Exists(previousWeights)) + inputWeights = previousWeights; + } + + string policy = ReadStringOption(args, "--policy", String.IsNullOrWhiteSpace(inputWeights) ? "random" : "trained-pv"); + string rolloutPath = Path.Combine(iterationDir, "selfplay.jsonl"); + string outputWeights = Path.Combine(iterationDir, "weights.json"); + int games = ReadOption(args, "--games", 100); + long seed = ReadLongOption(args, "--seed", 1); + int maxTurns = ReadOption(args, "--max-turns", 80); + int maxDecisions = ReadOption(args, "--max-decisions", 1000); + SearchConfig searchConfig = ReadSearchConfig(args); + + Console.WriteLine("Firebat Naxx self-play iteration"); + Console.WriteLine($"iteration={iterationName}, outputDir={iterationDir}"); + Console.WriteLine($"champion={championPath}"); + Console.WriteLine($"selfPlayPolicy={policy}, games={games}, seed={seed}"); + Console.WriteLine(SearchConfigSummary(searchConfig)); + if (!String.IsNullOrWhiteSpace(inputWeights)) + Console.WriteLine($"inputWeights={inputWeights}"); + + var rolloutConfig = new SelfPlayRolloutDatasetConfig + { + Games = games, + Seed = seed, + MaxTurns = maxTurns, + MaxDecisions = maxDecisions, + OutputPath = rolloutPath, + RecordTurnEndSnapshots = !HasFlag(args, "--turn-start-only"), + Policy = policy, + WeightsPath = inputWeights, + SearchConfig = searchConfig + }; + if (HasFlag(args, "--progress")) + { + rolloutConfig.ProgressIntervalGames = ReadOption(args, "--progress-every", 1); + rolloutConfig.Progress = PrintSelfPlayProgress; + } + + SelfPlayRolloutDatasetResult rollout = SelfPlayRolloutDatasetGenerator.Generate(rolloutConfig); + Console.WriteLine($"rollouts completed={rollout.CompletedGames}, unfinished={rollout.UnfinishedGames}, errors={rollout.Errors}, decisionCappedGames={rollout.DecisionCappedGames}"); + Console.WriteLine($"rolloutRows={rollout.RowsWritten}, valueRows={rollout.LegalValueRows + rollout.FullInfoValueRows}, hiddenRows={rollout.HiddenStateRows}, policyRows={rollout.PolicyRows}"); + Console.WriteLine($"rolloutFile={rollout.OutputPath}"); + + RunTrainer(args, rolloutPath, outputWeights); + + int benchmarkGames = ReadOption(args, "--benchmark-games", 10); + long validationSeed = ReadLongOption(args, "--validation-seed", seed + 100000); + Console.WriteLine(); + Console.WriteLine($"Validation benchmark: trained weights vs baseline, games={benchmarkGames}, seed={validationSeed}"); + Console.WriteLine($"weights={outputWeights}"); + FirebatNaxxBenchmarkRunner.PrintReport(FirebatNaxxBenchmarkRunner.RunTrainedPolicyValueBenchmark(outputWeights, benchmarkGames, validationSeed, searchConfig, maxTurns, HasFlag(args, "--verbose"))); + + if (String.IsNullOrWhiteSpace(inputWeights)) + { + CopyIfDifferent(outputWeights, championPath); + Console.WriteLine(); + Console.WriteLine($"No incumbent weights. Promoted {outputWeights} to champion."); + return; + } + + int arenaGames = ReadOption(args, "--arena-games", benchmarkGames); + double promotionThreshold = ReadDoubleOption(args, "--promotion-threshold", 0.55); + long arenaSeed = ReadLongOption(args, "--arena-seed", validationSeed + 500000); + Console.WriteLine(); + Console.WriteLine($"Arena gate: candidate vs incumbent, games={arenaGames}, seed={arenaSeed}, threshold={promotionThreshold:0.000}"); + FirebatNaxxBenchmarkReport arena = FirebatNaxxBenchmarkRunner.RunTrainedPolicyValueArena(outputWeights, inputWeights, arenaGames, arenaSeed, searchConfig, maxTurns, HasFlag(args, "--verbose")); + FirebatNaxxBenchmarkRunner.PrintReport(arena); + bool promoted = PrintArenaSummary(arena, promotionThreshold); + CopyIfDifferent(promoted ? outputWeights : inputWeights, championPath); + Console.WriteLine(promoted + ? $"Promoted candidate to champion: {championPath}" + : $"Kept incumbent champion: {championPath}"); + } + + private static void RunTrainer(string[] args, string rolloutPath, string outputWeights) + { + string python = ReadStringOption(args, "--python", "python3"); + var startInfo = new ProcessStartInfo + { + FileName = python, + WorkingDirectory = Environment.CurrentDirectory, + UseShellExecute = false + }; + startInfo.ArgumentList.Add(Path.Combine("tools", "train_firebat_value.py")); + startInfo.ArgumentList.Add(rolloutPath); + startInfo.ArgumentList.Add("--output"); + startInfo.ArgumentList.Add(outputWeights); + startInfo.ArgumentList.Add("--epochs"); + startInfo.ArgumentList.Add(ReadOption(args, "--epochs", 30).ToString()); + startInfo.ArgumentList.Add("--learning-rate"); + startInfo.ArgumentList.Add(ReadDoubleOption(args, "--learning-rate", 0.03).ToString(System.Globalization.CultureInfo.InvariantCulture)); + startInfo.ArgumentList.Add("--policy-epochs"); + startInfo.ArgumentList.Add(ReadOption(args, "--policy-epochs", ReadOption(args, "--epochs", 30)).ToString()); + startInfo.ArgumentList.Add("--policy-learning-rate"); + startInfo.ArgumentList.Add(ReadDoubleOption(args, "--policy-learning-rate", ReadDoubleOption(args, "--learning-rate", 0.03)).ToString(System.Globalization.CultureInfo.InvariantCulture)); + startInfo.ArgumentList.Add("--l2"); + startInfo.ArgumentList.Add(ReadDoubleOption(args, "--l2", 0.0001).ToString(System.Globalization.CultureInfo.InvariantCulture)); + startInfo.ArgumentList.Add("--holdout-mod"); + startInfo.ArgumentList.Add(ReadOption(args, "--holdout-mod", 5).ToString()); + startInfo.ArgumentList.Add("--seed"); + startInfo.ArgumentList.Add(ReadOption(args, "--train-seed", 1).ToString()); + + Console.WriteLine(); + Console.WriteLine("Training policy/value model"); + using (Process process = Process.Start(startInfo)) + { + process.WaitForExit(); + if (process.ExitCode != 0) + throw new InvalidOperationException($"Trainer failed with exit code {process.ExitCode}."); + } + } - // TEST BASIC AI + private static void PrintSelfPlayProgress(SelfPlayRolloutProgress progress) + { + SelfPlayRolloutDatasetResult result = progress.Result; + Console.WriteLine( + $"progress phase=selfplay games={progress.GamesProcessed}/{progress.Games} completed={result.CompletedGames} unfinished={result.UnfinishedGames} errors={result.Errors} decisionCapped={result.DecisionCappedGames} decisions={result.Decisions} actions={result.ActionsPlayed} policyRows={result.PolicyRows} rows={result.RowsWritten} nodes={result.NodesVisited} avgMoveMs={result.AverageMoveMs:0.0} elapsedSec={progress.Elapsed.TotalSeconds:0.0} lastSeed={progress.LastGameSeed}"); + Console.Out.Flush(); + } - //OneTurn(); - //FullGame(); - //RandomGames(); - //TestFullGames(); + private static bool PrintArenaSummary(FirebatNaxxBenchmarkReport report, double promotionThreshold) + { + int wins = FirebatNaxxBenchmarkRunner.CandidateArenaWins(report); + int games = FirebatNaxxBenchmarkRunner.CandidateArenaGames(report); + double rate = games == 0 ? 0 : wins / (double)games; + bool promoted = games > 0 && rate >= promotionThreshold; + Console.WriteLine(); + Console.WriteLine($"Arena candidate aggregate: {wins}/{games} wins ({rate * 100:0.0}%), threshold={promotionThreshold * 100:0.0}%, promote={(promoted ? "yes" : "no")}."); + return promoted; + } + + private static void CopyIfDifferent(string source, string destination) + { + string fullSource = Path.GetFullPath(source); + string fullDestination = Path.GetFullPath(destination); + if (fullSource.Equals(fullDestination, StringComparison.OrdinalIgnoreCase)) + return; + Directory.CreateDirectory(Path.GetDirectoryName(fullDestination)); + File.Copy(fullSource, fullDestination, overwrite: true); + } - Console.WriteLine("Test end!"); - Console.ReadLine(); + private static void RunShowGameCommand(string[] args) + { + Console.OutputEncoding = Encoding.UTF8; + var config = new FirebatNaxxTerminalViewerConfig + { + Seed = ReadLongOption(args, "--seed", 1), + MaxTurns = ReadOption(args, "--max-turns", 80), + MaxDecisions = ReadOption(args, "--max-decisions", 1000), + Policy = ReadStringOption(args, "--policy", "candidate"), + WeightsPath = ReadStringOption(args, "--weights", null), + EvaluationPolicy = ReadStringOption(args, "--eval-policy", null), + EvaluationWeightsPath = ReadStringOption(args, "--eval-weights", null), + ShowEvaluations = HasFlag(args, "--eval") || HasFlag(args, "--show-eval"), + EvaluationDecision = ReadOption(args, "--eval-decision", -1), + MaxEvaluationFeatures = ReadOption(args, "--eval-features", 0), + RenderAfterEachAction = HasFlag(args, "--step"), + SearchConfig = ReadSearchConfig(args) + }; + + FirebatNaxxTerminalViewer.Show(config, Console.Out); + } + + private static SearchConfig ReadSearchConfig(string[] args) + { + return new SearchConfig + { + MaxDepth = ReadOption(args, "--depth", 8), + MaxWidth = ReadOption(args, "--width", 80), + CandidateLines = ReadOption(args, "--lines", 12), + OpponentDepth = ReadOption(args, "--opponent-depth", 6), + OpponentResponseLines = ReadOption(args, "--opponent-lines", 8), + TimeBudgetMs = ReadOption(args, "--time-ms", 2000), + MctsIterations = ReadOption(args, "--mcts-sims", 160), + MctsExploration = ReadDoubleOption(args, "--mcts-cpuct", 1.5) + }; + } + + private static string SearchConfigSummary(SearchConfig config) + { + return $"depth={config.MaxDepth}, width={config.MaxWidth}, candidateLines={config.CandidateLines}, opponentDepth={config.OpponentDepth}, timeMs={config.TimeBudgetMs}, mctsSims={config.MctsIterations}, mctsCpuct={config.MctsExploration:0.###}"; + } + + private static int ReadOption(string[] args, string name, int fallback) + { + for (int i = 0; i < args.Length - 1; i++) + { + if (args[i].Equals(name, StringComparison.OrdinalIgnoreCase) && int.TryParse(args[i + 1], out int value)) + return value; + } + return fallback; + } + + private static long ReadLongOption(string[] args, string name, long fallback) + { + for (int i = 0; i < args.Length - 1; i++) + { + if (args[i].Equals(name, StringComparison.OrdinalIgnoreCase) && long.TryParse(args[i + 1], out long value)) + return value; + } + return fallback; + } + + private static string ReadStringOption(string[] args, string name, string fallback) + { + for (int i = 0; i < args.Length - 1; i++) + { + if (args[i].Equals(name, StringComparison.OrdinalIgnoreCase)) + return args[i + 1]; + } + return fallback; + } + + private static double ReadDoubleOption(string[] args, string name, double fallback) + { + for (int i = 0; i < args.Length - 1; i++) + { + if (args[i].Equals(name, StringComparison.OrdinalIgnoreCase) && double.TryParse(args[i + 1], System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture, out double value)) + return value; + } + return fallback; + } + + private static bool HasFlag(string[] args, string name) + { + return args.Any(arg => arg.Equals(name, StringComparison.OrdinalIgnoreCase)); } public static void RandomGames() diff --git a/core-extensions/SabberStoneBasicAI/src/Score/ComboDruidScore.cs b/core-extensions/SabberStoneBasicAI/src/Score/ComboDruidScore.cs new file mode 100644 index 000000000..817793f03 --- /dev/null +++ b/core-extensions/SabberStoneBasicAI/src/Score/ComboDruidScore.cs @@ -0,0 +1,81 @@ +#region copyright +// SabberStone, Hearthstone Simulator in C# .NET Core +// Copyright (C) 2017-2019 SabberStone Team, darkfriend77 & rnilva +// +// SabberStone is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License. +// SabberStone is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +#endregion +using System; +using System.Collections.Generic; +using System.Linq; +using SabberStoneCore.Model.Entities; + +namespace SabberStoneBasicAI.Score +{ + public class ComboDruidScore : Score + { + public override int Rate() + { + if (OpHeroHp < 1) + return int.MaxValue; + + if (HeroHp < 1) + return int.MinValue; + + int boardAttack = MinionTotAtk + HeroAtk; + int opponentBoardAttack = OpMinionTotAtk + OpHeroAtk; + int comboThreat = HasInHand("Force of Nature") && HasInHand("Savage Roar") ? 2500 : 0; + int lethalPressure = Math.Max(0, 14 - OpHeroHp) * 250; + + int result = 0; + result += (HeroHp - OpHeroHp) * 25; + result += (MinionTotHealth - OpMinionTotHealth) * 20; + result += (boardAttack - opponentBoardAttack) * 45; + result += BoardZone.Count * 45; + result -= OpBoardZone.Count * 60; + result -= OpMinionTotAtk * 90; + result -= OpMinionTotHealthTaunt * 250; + result += Controller.BaseMana * 65; + result += HandCnt * 30; + result += comboThreat; + result += lethalPressure; + + return result; + } + + public override Func, List> MulliganRule() + { + return cards => cards + .Where(card => KeepInOpeningHand(card.Card.Name, card.Cost)) + .Select(card => card.Id) + .ToList(); + } + + private static bool KeepInOpeningHand(string name, int cost) + { + switch (name) + { + case "Innervate": + case "Wild Growth": + case "Wrath": + case "Shade of Naxxramas": + case "Keeper of the Grove": + case "Swipe": + return true; + default: + return cost <= 2; + } + } + + private bool HasInHand(string name) + { + return Hand.Any(card => card.Card.Name == name); + } + } +} diff --git a/core-extensions/SabberStoneBasicAI/src/Score/ZooWarlockScore.cs b/core-extensions/SabberStoneBasicAI/src/Score/ZooWarlockScore.cs new file mode 100644 index 000000000..6b1e4cfe6 --- /dev/null +++ b/core-extensions/SabberStoneBasicAI/src/Score/ZooWarlockScore.cs @@ -0,0 +1,82 @@ +#region copyright +// SabberStone, Hearthstone Simulator in C# .NET Core +// Copyright (C) 2017-2019 SabberStone Team, darkfriend77 & rnilva +// +// SabberStone is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License. +// SabberStone is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +#endregion +using System; +using System.Collections.Generic; +using System.Linq; +using SabberStoneCore.Model.Entities; + +namespace SabberStoneBasicAI.Score +{ + public class ZooWarlockScore : Score + { + public override int Rate() + { + if (OpHeroHp < 1) + return int.MaxValue; + + if (HeroHp < 1) + return int.MinValue; + + int boardAttack = MinionTotAtk + HeroAtk; + int opponentBoardAttack = OpMinionTotAtk + OpHeroAtk; + int handBurn = CountInHand("Soulfire") * 4 + CountInHand("Power Overwhelming") * 4 + CountInHand("Doomguard") * 5; + int lethalPressure = Math.Max(0, boardAttack + handBurn - OpHeroHp) * 1000; + + int result = 0; + result += (30 - OpHeroHp) * 90; + result += (HeroHp - 10) * 20; + result += BoardZone.Count * 160; + result -= OpBoardZone.Count * 75; + result += (boardAttack - opponentBoardAttack) * 80; + result += MinionTotHealth * 25; + result -= OpMinionTotHealthTaunt * 350; + result += HandCnt * 15; + result += lethalPressure; + + return result; + } + + public override Func, List> MulliganRule() + { + return cards => cards + .Where(card => KeepInOpeningHand(card.Card.Name, card.Cost)) + .Select(card => card.Id) + .ToList(); + } + + private static bool KeepInOpeningHand(string name, int cost) + { + switch (name) + { + case "Flame Imp": + case "Voidwalker": + case "Abusive Sergeant": + case "Leper Gnome": + case "Undertaker": + case "Haunted Creeper": + case "Knife Juggler": + case "Nerubian Egg": + case "Dire Wolf Alpha": + return true; + default: + return cost <= 2 && name != "Soulfire" && name != "Power Overwhelming"; + } + } + + private int CountInHand(string name) + { + return Hand.Count(card => card.Card.Name == name); + } + } +} diff --git a/tools/firebat_checkpoint_scheduler.py b/tools/firebat_checkpoint_scheduler.py new file mode 100644 index 000000000..e99017424 --- /dev/null +++ b/tools/firebat_checkpoint_scheduler.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +import argparse +import json +import subprocess +import sys +import time +from datetime import datetime +from pathlib import Path +from urllib import request, error + + +REPO_ROOT = Path(__file__).resolve().parents[1] +LOG_DIR = REPO_ROOT / "data" / "firebat-dashboard" + + +def dashboard_json(path, payload=None): + url = f"http://127.0.0.1:8765{path}" + if payload is None: + return json.load(request.urlopen(url)) + body = json.dumps(payload).encode("utf-8") + req = request.Request(url, data=body, headers={"Content-Type": "application/json"}, method="POST") + return json.load(request.urlopen(req)) + + +def wait_until_idle(log, poll_seconds): + while True: + status = dashboard_json("/api/status") + if not status.get("running"): + return status + progress = status.get("metrics", {}).get("progress", {}) + log.write( + f"waiting run={status.get('run', {}).get('id')} " + f"games={progress.get('games', 0)}/{progress.get('gamesTotal', 0)} " + f"errors={progress.get('errors', 0)}\n" + ) + log.flush() + time.sleep(poll_seconds) + + +def run_schedule(args): + LOG_DIR.mkdir(parents=True, exist_ok=True) + schedule_id = datetime.now().strftime("%Y%m%d-%H%M%S") + log_path = LOG_DIR / f"checkpoint-schedule-{schedule_id}.log" + chunks = [] + remaining = args.total_games + while remaining > 0: + count = min(args.checkpoint_games, remaining) + chunks.append(count) + remaining -= count + + with open(log_path, "a", buffering=1, encoding="utf-8") as log: + log.write( + f"schedule={schedule_id} totalGames={args.total_games} checkpointGames={args.checkpoint_games} " + f"chunks={len(chunks)} workers={args.workers}\n" + ) + completed = 0 + for index, games in enumerate(chunks, start=1): + wait_until_idle(log, args.poll_seconds) + payload = { + "outputDir": args.output_dir, + "policy": args.policy, + "games": games, + "workers": args.workers, + "mctsSims": args.mcts_sims, + "mctsCpuct": args.mcts_cpuct, + "depth": args.depth, + "width": args.width, + "timeMs": args.time_ms, + "epochs": args.epochs, + "policyEpochs": args.policy_epochs, + "benchmarkGames": args.benchmark_games, + "arenaGames": args.arena_games, + } + log.write(f"starting chunk={index}/{len(chunks)} games={games} completedBefore={completed}\n") + try: + run = dashboard_json("/api/start", payload) + except error.HTTPError as exc: + message = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"dashboard start failed: {message}") from exc + log.write(f"run={run.get('id')} iteration={run.get('iteration')} pid={run.get('pid')}\n") + while True: + status = dashboard_json("/api/status") + progress = status.get("metrics", {}).get("progress", {}) + log.write( + f"progress chunk={index}/{len(chunks)} run={status.get('run', {}).get('id')} " + f"games={progress.get('games', 0)}/{progress.get('gamesTotal', 0)} " + f"errors={progress.get('errors', 0)} unfinished={progress.get('unfinished', 0)}\n" + ) + if not status.get("running"): + run_info = status.get("run") or {} + log.write( + f"finished chunk={index}/{len(chunks)} run={run_info.get('id')} " + f"returnCode={run_info.get('returnCode')} metrics={status.get('files', {}).get('metrics', {}).get('path')}\n" + ) + break + log.flush() + time.sleep(args.poll_seconds) + completed += games + log.write(f"complete schedule={schedule_id} totalGames={completed}\n") + return log_path + + +def daemonize(args): + LOG_DIR.mkdir(parents=True, exist_ok=True) + child_args = [arg for arg in sys.argv[1:] if arg != "--daemon"] + launcher_log = LOG_DIR / "checkpoint-scheduler-launcher.log" + with open(launcher_log, "a", buffering=1, encoding="utf-8") as log: + process = subprocess.Popen( + [sys.executable, str(Path(__file__).resolve()), *child_args], + cwd=str(REPO_ROOT), + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + text=True, + ) + print(json.dumps({"pid": process.pid, "log": str(launcher_log)})) + + +def parse_args(): + parser = argparse.ArgumentParser(description="Schedule repeated Firebat dashboard self-play checkpoints.") + parser.add_argument("--daemon", action="store_true") + parser.add_argument("--total-games", type=int, default=20000) + parser.add_argument("--checkpoint-games", type=int, default=1000) + parser.add_argument("--workers", type=int, default=8) + parser.add_argument("--output-dir", default="data/iterations") + parser.add_argument("--policy", default="trained-pv") + parser.add_argument("--mcts-sims", type=int, default=80) + parser.add_argument("--mcts-cpuct", type=float, default=1.5) + parser.add_argument("--depth", type=int, default=4) + parser.add_argument("--width", type=int, default=32) + parser.add_argument("--time-ms", type=int, default=1000) + parser.add_argument("--epochs", type=int, default=8) + parser.add_argument("--policy-epochs", type=int, default=8) + parser.add_argument("--benchmark-games", type=int, default=100) + parser.add_argument("--arena-games", type=int, default=100) + parser.add_argument("--poll-seconds", type=int, default=15) + return parser.parse_args() + + +def main(): + args = parse_args() + if args.checkpoint_games < 1 or args.total_games < 1: + raise SystemExit("total-games and checkpoint-games must be positive") + if args.daemon: + daemonize(args) + return + log_path = run_schedule(args) + print(json.dumps({"log": str(log_path)})) + + +if __name__ == "__main__": + main() diff --git a/tools/firebat_selfplay_dashboard.py b/tools/firebat_selfplay_dashboard.py new file mode 100644 index 000000000..13a8eed43 --- /dev/null +++ b/tools/firebat_selfplay_dashboard.py @@ -0,0 +1,1325 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import re +import signal +import shutil +import subprocess +import sys +import threading +import time +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import urlparse + + +REPO_ROOT = Path(__file__).resolve().parents[1] +PROJECT = REPO_ROOT / "core-extensions" / "SabberStoneBasicAI" / "SabberStoneBasicAI.csproj" +DASH_ROOT = REPO_ROOT / "data" / "firebat-dashboard" +DEFAULT_ITERATION_ROOT = REPO_ROOT / "data" / "iterations" + + +def utc_now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def resolve_path(value, fallback): + if value is None or str(value).strip() == "": + return Path(fallback) + path = Path(str(value)).expanduser() + return path if path.is_absolute() else REPO_ROOT / path + + +def next_iteration(output_dir): + output_dir.mkdir(parents=True, exist_ok=True) + highest = 0 + for child in output_dir.iterdir(): + match = re.fullmatch(r"iter-(\d{3,})", child.name) + if match: + highest = max(highest, int(match.group(1))) + return highest + 1 + + +def as_int(data, key, fallback): + try: + return int(data.get(key, fallback)) + except (TypeError, ValueError): + return fallback + + +def as_float(data, key, fallback): + try: + return float(data.get(key, fallback)) + except (TypeError, ValueError): + return fallback + + +def read_tail(path, limit=70000): + if not path or not Path(path).exists(): + return "" + with open(path, "rb") as handle: + handle.seek(0, os.SEEK_END) + size = handle.tell() + handle.seek(max(0, size - limit), os.SEEK_SET) + return handle.read().decode("utf-8", errors="replace") + + +def file_info(path): + path = Path(path) + if not path.exists(): + return None + stat = path.stat() + return { + "path": str(path), + "bytes": stat.st_size, + "modified": datetime.fromtimestamp(stat.st_mtime, timezone.utc).isoformat(timespec="seconds"), + } + + +def pid_running(pid): + try: + os.kill(int(pid), 0) + return True + except (OSError, TypeError, ValueError): + return False + + +def parse_key_values(line): + result = {} + for key, value in re.findall(r"([A-Za-z][A-Za-z0-9]*)=([^,\s]+)", line): + if "/" in value: + left, right = value.split("/", 1) + if left.isdigit() and right.isdigit(): + result[key] = int(left) + result[key + "Total"] = int(right) + continue + try: + result[key] = int(value) + continue + except ValueError: + pass + try: + result[key] = float(value) + continue + except ValueError: + result[key] = value + return result + + +def parse_log(text, running): + metrics = { + "phase": "running" if running else "idle", + "progress": {}, + "training": [], + "benchmarks": [], + } + for line in text.splitlines(): + if line.startswith("progress phase=selfplay"): + metrics["phase"] = "self-play" + metrics["progress"] = parse_key_values(line) + elif line.startswith("Training policy/value model"): + metrics["phase"] = "training" + elif re.match(r"^(legal_value|full_info_value|legal_policy):", line): + metrics["phase"] = "training" + metrics["training"].append(line) + elif line.startswith("Validation benchmark:"): + metrics["phase"] = "validation" + metrics["benchmarks"].append(line) + elif "arena" in line.lower() or "wins (" in line: + metrics["benchmarks"].append(line) + elif line.startswith("No incumbent weights.") or line.startswith("Promoted ") or line.startswith("Kept incumbent"): + metrics["phase"] = "complete" + metrics["benchmarks"].append(line) + if not running and metrics["phase"] in ("running", "self-play", "training", "validation"): + metrics["phase"] = "complete" + return metrics + + +def split_games(total, workers): + workers = max(1, min(int(workers), int(total))) + base = total // workers + rem = total % workers + counts = [] + start = 0 + for index in range(workers): + count = base + (1 if index < rem else 0) + counts.append((index, start, count)) + start += count + return counts + + +def latest_progress_from_log(path): + progress = {} + for line in read_tail(path, 200000).splitlines(): + if line.startswith("progress phase=selfplay"): + progress = parse_key_values(line) + return progress + + +def merge_jsonl(shards, output_path): + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", encoding="utf-8") as out: + for shard in shards: + if not shard.exists(): + continue + with open(shard, "r", encoding="utf-8") as inp: + shutil.copyfileobj(inp, out) + + +def run_and_log(cmd, log, cwd=REPO_ROOT): + log.write("$ " + " ".join(str(part) for part in cmd) + "\n") + log.flush() + process = subprocess.Popen( + [str(part) for part in cmd], + cwd=str(cwd), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + lines = [] + for line in process.stdout: + lines.append(line.rstrip("\n")) + log.write(line) + log.flush() + return_code = process.wait() + if return_code != 0: + raise RuntimeError(f"command failed with exit code {return_code}: {' '.join(str(part) for part in cmd)}") + return "\n".join(lines) + + +def parse_training_lines(lines): + metrics = {"lines": lines} + for line in lines: + match = re.match( + r"^(legal_value|full_info_value|legal_policy):([a-z_]+) train=(\d+) validation=(\d+) (.+)$", + line, + ) + if not match: + continue + kind, side, train_count, validation_count, rest = match.groups() + entry = { + "kind": kind, + "side": side, + "train": int(train_count), + "validation": int(validation_count), + } + for key, value in re.findall(r"([a-zA-Z]+)=([0-9.]+)", rest): + entry[key] = float(value) + metrics.setdefault("entries", []).append(entry) + + entries = metrics.get("entries", []) + def avg(kind, key): + values = [entry[key] for entry in entries if entry.get("kind") == kind and key in entry] + return sum(values) / len(values) if values else None + + metrics["legalPolicyValidationAcc"] = avg("legal_policy", "validationAcc") + metrics["legalPolicyValidationNll"] = avg("legal_policy", "validationNll") + metrics["legalValueValidationMse"] = avg("legal_value", "validationMse") + metrics["fullInfoValueValidationMse"] = avg("full_info_value", "validationMse") + return metrics + + +def parse_benchmark_report(text): + scenarios = [] + current = None + aggregate = None + arena = None + lines = [line.strip() for line in text.splitlines()] + scenario_names = { + "baseline mirror", + "trained policy-value druid vs baseline zoo", + "baseline druid vs trained policy-value zoo", + "trained policy-value mirror", + "candidate druid vs incumbent zoo", + "incumbent druid vs candidate zoo", + "candidate mirror", + "incumbent mirror", + } + for line in lines: + if line in scenario_names: + current = {"name": line} + scenarios.append(current) + continue + if current: + p1 = re.match(r"^Firebat Combo Druid \((.*?)\): (\d+)/(\d+) wins \(([0-9.]+)%\)", line) + if p1: + current["player1Policy"] = p1.group(1) + current["player1Wins"] = int(p1.group(2)) + current["games"] = int(p1.group(3)) + current["player1WinRate"] = float(p1.group(4)) / 100.0 + continue + p2 = re.match(r"^Firebat Zoo Warlock \((.*?)\): (\d+)/(\d+) wins \(([0-9.]+)%\)", line) + if p2: + current["player2Policy"] = p2.group(1) + current["player2Wins"] = int(p2.group(2)) + current["games"] = int(p2.group(3)) + current["player2WinRate"] = float(p2.group(4)) / 100.0 + continue + details = re.match(r"^ties=(\d+), unfinished=(\d+), errors=(\d+), avgTurns=([0-9.]+), avgMoveMs=([0-9.]+), nodes=(\d+)", line) + if details: + current["ties"] = int(details.group(1)) + current["unfinished"] = int(details.group(2)) + current["errors"] = int(details.group(3)) + current["avgTurns"] = float(details.group(4)) + current["avgMoveMs"] = float(details.group(5)) + current["nodes"] = int(details.group(6)) + continue + candidate = re.match(r"^Candidate vs baseline aggregate: (\d+)/(\d+) wins \(([0-9.]+)%\)", line) + if candidate: + aggregate = { + "wins": int(candidate.group(1)), + "games": int(candidate.group(2)), + "winRate": float(candidate.group(3)) / 100.0, + } + continue + arena_match = re.match(r"^Arena candidate aggregate: (\d+)/(\d+) wins \(([0-9.]+)%\), threshold=([0-9.]+)%, promote=(yes|no)", line) + if arena_match: + arena = { + "wins": int(arena_match.group(1)), + "games": int(arena_match.group(2)), + "winRate": float(arena_match.group(3)) / 100.0, + "threshold": float(arena_match.group(4)) / 100.0, + "promoted": arena_match.group(5) == "yes", + } + + return { + "scenarios": scenarios, + "candidateVsBaseline": aggregate, + "candidateVsChampion": arena, + "raw": text, + } + + +def write_metrics(path, metrics): + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", encoding="utf-8") as handle: + json.dump(metrics, handle, indent=2) + + +def load_history(output_dir): + output_dir = resolve_path(output_dir, DEFAULT_ITERATION_ROOT) + rows = [] + for metrics_path in sorted(output_dir.glob("iter-*/metrics.json")): + try: + with open(metrics_path, "r", encoding="utf-8") as handle: + row = json.load(handle) + except (OSError, json.JSONDecodeError): + continue + row["metricsPath"] = str(metrics_path) + rows.append(row) + cumulative = 0 + for row in sorted(rows, key=lambda item: item.get("iteration", 0)): + cumulative += int(row.get("selfPlay", {}).get("games", 0)) + row["cumulativeGames"] = cumulative + return rows + + +def run_parallel_job(config_path): + with open(config_path, "r", encoding="utf-8") as handle: + config = json.load(handle) + + run_dir = Path(config["runDir"]) + iteration_dir = Path(config["iterationDir"]) + output_dir = Path(config["outputDir"]) + shard_dir = iteration_dir / "shards" + log_path = Path(config["logPath"]) + selfplay_path = iteration_dir / "selfplay.jsonl" + weights_path = iteration_dir / "weights.json" + champion_path = Path(config["championPath"]) + run_dir.mkdir(parents=True, exist_ok=True) + iteration_dir.mkdir(parents=True, exist_ok=True) + shard_dir.mkdir(parents=True, exist_ok=True) + + with open(log_path, "a", buffering=1, encoding="utf-8") as log: + log.write("Firebat Naxx parallel self-play iteration\n") + log.write(f"iteration=iter-{int(config['iteration']):03d}, outputDir={iteration_dir}\n") + log.write(f"champion={champion_path}\n") + log.write(f"selfPlayPolicy={config['policy']}, games={config['games']}, seed={config['seed']}, workers={config['workers']}\n") + log.write( + f"depth={config['depth']}, width={config['width']}, timeMs={config['timeMs']}, " + f"mctsSims={config['mctsSims']}, mctsCpuct={config['mctsCpuct']}\n" + ) + if config.get("inputWeights"): + log.write(f"inputWeights={config['inputWeights']}\n") + + shard_specs = split_games(int(config["games"]), int(config["workers"])) + processes = [] + shard_paths = [] + started = time.time() + for index, offset, count in shard_specs: + shard_path = shard_dir / f"selfplay-shard-{index:02d}.jsonl" + shard_log_path = shard_dir / f"worker-{index:02d}.log" + shard_paths.append(shard_path) + cmd = [ + "dotnet", "run", + "--project", str(PROJECT), + "-f", "net10.0", + "--no-build", + "--", + "generate-selfplay-rollouts", + "--games", str(count), + "--seed", str(int(config["seed"]) + offset), + "--output", str(shard_path), + "--policy", str(config["policy"]), + "--max-turns", str(config["maxTurns"]), + "--max-decisions", str(config["maxDecisions"]), + "--depth", str(config["depth"]), + "--width", str(config["width"]), + "--mcts-sims", str(config["mctsSims"]), + "--mcts-cpuct", str(config["mctsCpuct"]), + "--time-ms", str(config["timeMs"]), + "--progress", + "--progress-every", str(config.get("progressEvery", 10)), + ] + if config.get("inputWeights"): + cmd.extend(["--weights", str(config["inputWeights"])]) + shard_log = open(shard_log_path, "w", buffering=1, encoding="utf-8") + shard_log.write("$ " + " ".join(cmd) + "\n") + process = subprocess.Popen(cmd, cwd=str(REPO_ROOT), stdout=shard_log, stderr=subprocess.STDOUT, text=True) + processes.append({"index": index, "count": count, "path": shard_path, "logPath": shard_log_path, "log": shard_log, "process": process}) + + last_reported_games = -1 + while True: + aggregate = { + "games": 0, + "completed": 0, + "unfinished": 0, + "errors": 0, + "decisionCapped": 0, + "decisions": 0, + "actions": 0, + "policyRows": 0, + "rows": 0, + "nodes": 0, + "avgMoveMsWeighted": 0.0, + } + running = False + for worker in processes: + if worker["process"].poll() is None: + running = True + progress = latest_progress_from_log(worker["logPath"]) + if progress: + decisions = int(progress.get("decisions", 0)) + aggregate["games"] += int(progress.get("games", 0)) + aggregate["completed"] += int(progress.get("completed", 0)) + aggregate["unfinished"] += int(progress.get("unfinished", 0)) + aggregate["errors"] += int(progress.get("errors", 0)) + aggregate["decisionCapped"] += int(progress.get("decisionCapped", 0)) + aggregate["decisions"] += decisions + aggregate["actions"] += int(progress.get("actions", 0)) + aggregate["policyRows"] += int(progress.get("policyRows", 0)) + aggregate["rows"] += int(progress.get("rows", 0)) + aggregate["nodes"] += int(progress.get("nodes", 0)) + aggregate["avgMoveMsWeighted"] += float(progress.get("avgMoveMs", 0)) * decisions + if aggregate["games"] != last_reported_games: + avg = aggregate["avgMoveMsWeighted"] / aggregate["decisions"] if aggregate["decisions"] else 0 + elapsed = time.time() - started + log.write( + f"progress phase=selfplay games={aggregate['games']}/{config['games']} workers={config['workers']} " + f"completed={aggregate['completed']} unfinished={aggregate['unfinished']} errors={aggregate['errors']} " + f"decisionCapped={aggregate['decisionCapped']} decisions={aggregate['decisions']} actions={aggregate['actions']} " + f"policyRows={aggregate['policyRows']} rows={aggregate['rows']} nodes={aggregate['nodes']} " + f"avgMoveMs={avg:0.1f} elapsedSec={elapsed:0.1f}\n" + ) + last_reported_games = aggregate["games"] + if not running: + break + time.sleep(2) + + for worker in processes: + worker["log"].close() + rc = worker["process"].wait() + if rc != 0: + raise RuntimeError(f"worker {worker['index']} failed with exit code {rc}; see {worker['logPath']}") + + elapsed = time.time() - started + log.write(f"mergingShards={len(shard_paths)} output={selfplay_path}\n") + merge_jsonl(shard_paths, selfplay_path) + if not config.get("keepShards"): + for shard_path in shard_paths: + try: + shard_path.unlink() + except OSError: + pass + + trainer_cmd = [ + sys.executable, + str(REPO_ROOT / "tools" / "train_firebat_value.py"), + str(selfplay_path), + "--output", str(weights_path), + "--epochs", str(config["epochs"]), + "--policy-epochs", str(config["policyEpochs"]), + "--learning-rate", str(config["learningRate"]), + "--policy-learning-rate", str(config["policyLearningRate"]), + "--l2", str(config["l2"]), + "--holdout-mod", str(config["holdoutMod"]), + "--seed", str(config["trainSeed"]), + ] + log.write("\nTraining policy/value model\n") + training_text = run_and_log(trainer_cmd, log) + training_lines = [line for line in training_text.splitlines() if re.match(r"^(legal_value|full_info_value|legal_policy):", line)] + + validation_cmd = [ + "dotnet", "run", + "--project", str(PROJECT), + "-f", "net10.0", + "--no-build", + "--", + "pv-benchmark", + "--weights", str(weights_path), + "--games", str(config["benchmarkGames"]), + "--seed", str(config["validationSeed"]), + "--max-turns", str(config["maxTurns"]), + "--depth", str(config["depth"]), + "--width", str(config["width"]), + "--mcts-sims", str(config["mctsSims"]), + "--mcts-cpuct", str(config["mctsCpuct"]), + "--time-ms", str(config["timeMs"]), + ] + log.write("\nValidation benchmark\n") + validation_text = run_and_log(validation_cmd, log) + + arena_text = "" + promoted = False + if config.get("inputWeights"): + arena_cmd = [ + "dotnet", "run", + "--project", str(PROJECT), + "-f", "net10.0", + "--no-build", + "--", + "pv-arena", + "--candidate", str(weights_path), + "--incumbent", str(config["inputWeights"]), + "--games", str(config["arenaGames"]), + "--seed", str(config["arenaSeed"]), + "--promotion-threshold", str(config["promotionThreshold"]), + "--max-turns", str(config["maxTurns"]), + "--depth", str(config["depth"]), + "--width", str(config["width"]), + "--mcts-sims", str(config["mctsSims"]), + "--mcts-cpuct", str(config["mctsCpuct"]), + "--time-ms", str(config["timeMs"]), + ] + log.write("\nArena gate\n") + arena_text = run_and_log(arena_cmd, log) + arena_report = parse_benchmark_report(arena_text) + promoted = bool((arena_report.get("candidateVsChampion") or {}).get("promoted")) + shutil.copyfile(weights_path if promoted else Path(config["inputWeights"]), champion_path) + log.write(f"{'Promoted candidate' if promoted else 'Kept incumbent'} champion: {champion_path}\n") + else: + shutil.copyfile(weights_path, champion_path) + promoted = True + log.write(f"No incumbent weights. Promoted {weights_path} to champion.\n") + + validation_report = parse_benchmark_report(validation_text) + arena_report = parse_benchmark_report(arena_text) if arena_text else None + selfplay_info = latest_progress_from_log(log_path) + metrics = { + "schema": "firebat_naxx_iteration_metrics_v1", + "createdAt": utc_now(), + "iteration": int(config["iteration"]), + "iterationName": f"iter-{int(config['iteration']):03d}", + "weightsPath": str(weights_path), + "championPath": str(champion_path), + "inputWeightsPath": config.get("inputWeights"), + "selfPlay": { + "games": int(config["games"]), + "workers": int(config["workers"]), + "policy": config["policy"], + "seed": int(config["seed"]), + "completed": int(selfplay_info.get("completed", 0)), + "unfinished": int(selfplay_info.get("unfinished", 0)), + "errors": int(selfplay_info.get("errors", 0)), + "decisionCapped": int(selfplay_info.get("decisionCapped", 0)), + "decisions": int(selfplay_info.get("decisions", 0)), + "actions": int(selfplay_info.get("actions", 0)), + "policyRows": int(selfplay_info.get("policyRows", 0)), + "rows": int(selfplay_info.get("rows", 0)), + "nodes": int(selfplay_info.get("nodes", 0)), + "avgMoveMs": float(selfplay_info.get("avgMoveMs", 0)), + "elapsedSec": float(selfplay_info.get("elapsedSec", elapsed)), + "gamesPerSec": int(config["games"]) / max(1.0, float(selfplay_info.get("elapsedSec", elapsed))), + }, + "training": parse_training_lines(training_lines), + "validation": validation_report, + "arena": arena_report, + "promoted": promoted, + } + write_metrics(iteration_dir / "metrics.json", metrics) + log.write(f"metrics={iteration_dir / 'metrics.json'}\n") + + +class DashboardState: + def __init__(self): + self.lock = threading.Lock() + self.process = None + self.log_handle = None + self.run = self._load_active() + + def is_running_locked(self): + if self.process is not None: + return self.process.poll() is None + return bool(self.run and self.run.get("pid") and pid_running(self.run.get("pid"))) + + def _load_active(self): + path = DASH_ROOT / "active.json" + if not path.exists(): + return None + try: + with open(path, "r", encoding="utf-8") as handle: + return json.load(handle) + except (OSError, json.JSONDecodeError): + return None + + def start(self, params): + with self.lock: + if self.is_running_locked(): + raise RuntimeError("self-play is already running") + + output_dir = resolve_path(params.get("outputDir"), DEFAULT_ITERATION_ROOT) + iteration = as_int(params, "iteration", next_iteration(output_dir)) + seed = as_int(params, "seed", int(time.time()) % 1000000) + games = as_int(params, "games", 25) + workers = max(1, as_int(params, "workers", 1)) + policy = str(params.get("policy", "mcts") or "mcts") + run_id = f"{datetime.now().strftime('%Y%m%d-%H%M%S')}-iter-{iteration:03d}" + run_dir = DASH_ROOT / "runs" / run_id + run_dir.mkdir(parents=True, exist_ok=True) + log_path = run_dir / "self-play.log" + validation_seed = as_int(params, "validationSeed", seed + 100000) + + champion_path = output_dir / "champion.json" + input_weights = str(params.get("weights") or "").strip() + if not input_weights and champion_path.exists(): + input_weights = str(champion_path) + elif not input_weights and iteration > 1: + previous_weights = output_dir / f"iter-{iteration - 1:03d}" / "weights.json" + if previous_weights.exists(): + input_weights = str(previous_weights) + if policy.startswith("trained") and not input_weights: + policy = "mcts" + + job_config = { + "runId": run_id, + "runDir": str(run_dir), + "logPath": str(log_path), + "outputDir": str(output_dir), + "iteration": iteration, + "iterationDir": str(output_dir / f"iter-{iteration:03d}"), + "championPath": str(champion_path), + "inputWeights": input_weights or None, + "policy": policy, + "games": games, + "workers": workers, + "seed": seed, + "validationSeed": validation_seed, + "arenaSeed": as_int(params, "arenaSeed", validation_seed + 500000), + "benchmarkGames": as_int(params, "benchmarkGames", 4), + "arenaGames": as_int(params, "arenaGames", as_int(params, "benchmarkGames", 4)), + "promotionThreshold": as_float(params, "promotionThreshold", 0.55), + "epochs": as_int(params, "epochs", 6), + "policyEpochs": as_int(params, "policyEpochs", as_int(params, "epochs", 6)), + "learningRate": as_float(params, "learningRate", 0.03), + "policyLearningRate": as_float(params, "policyLearningRate", as_float(params, "learningRate", 0.03)), + "l2": as_float(params, "l2", 0.0001), + "holdoutMod": as_int(params, "holdoutMod", 5), + "trainSeed": as_int(params, "trainSeed", 1), + "maxTurns": as_int(params, "maxTurns", 80), + "maxDecisions": as_int(params, "maxDecisions", 1000), + "depth": as_int(params, "depth", 4), + "width": as_int(params, "width", 32), + "mctsSims": as_int(params, "mctsSims", 80), + "mctsCpuct": as_float(params, "mctsCpuct", 1.5), + "timeMs": as_int(params, "timeMs", 1000), + "progressEvery": as_int(params, "progressEvery", 10), + } + job_path = run_dir / "job.json" + with open(job_path, "w", encoding="utf-8") as handle: + json.dump(job_config, handle, indent=2) + + cmd = [sys.executable, str(Path(__file__).resolve()), "--run-job", str(job_path)] + + self.log_handle = open(log_path, "w", buffering=1, encoding="utf-8") + self.process = subprocess.Popen( + cmd, + cwd=str(REPO_ROOT), + stdout=self.log_handle, + stderr=subprocess.STDOUT, + text=True, + ) + self.run = { + "id": run_id, + "pid": self.process.pid, + "startedAt": utc_now(), + "finishedAt": None, + "returnCode": None, + "command": cmd, + "logPath": str(log_path), + "runDir": str(run_dir), + "outputDir": str(output_dir), + "iteration": iteration, + "iterationDir": str(output_dir / f"iter-{iteration:03d}"), + "championPath": str(output_dir / "champion.json"), + "games": games, + "workers": workers, + "policy": policy, + } + self._write_active_locked() + threading.Thread(target=self._watch, daemon=True).start() + return self.run + + def stop(self): + with self.lock: + if not self.is_running_locked(): + return False + if self.process is not None: + self.process.terminate() + elif self.run and self.run.get("pid"): + os.kill(int(self.run["pid"]), signal.SIGTERM) + return True + + def _watch(self): + process = self.process + return_code = process.wait() + with self.lock: + if self.run is not None and self.process is process: + self.run["returnCode"] = return_code + self.run["finishedAt"] = utc_now() + self._write_active_locked() + if self.log_handle is not None: + self.log_handle.close() + self.log_handle = None + + def _write_active_locked(self): + DASH_ROOT.mkdir(parents=True, exist_ok=True) + with open(DASH_ROOT / "active.json", "w", encoding="utf-8") as handle: + json.dump(self.run, handle, indent=2) + + def status(self): + with self.lock: + running = self.is_running_locked() + run = dict(self.run) if self.run else None + log = read_tail(run.get("logPath")) if run else "" + metrics = parse_log(log, running) + files = {} + if run: + iteration_dir = Path(run["iterationDir"]) + files = { + "selfplay": file_info(iteration_dir / "selfplay.jsonl"), + "weights": file_info(iteration_dir / "weights.json"), + "metrics": file_info(iteration_dir / "metrics.json"), + "champion": file_info(run["championPath"]), + "log": file_info(run["logPath"]), + } + history_dir = run.get("outputDir") if run else DEFAULT_ITERATION_ROOT + return { + "running": running, + "run": run, + "metrics": metrics, + "files": files, + "history": load_history(history_dir), + "logTail": log.splitlines()[-220:], + "defaults": default_start_params(), + } + + +STATE = DashboardState() + + +def default_start_params(): + return { + "outputDir": str(DEFAULT_ITERATION_ROOT.relative_to(REPO_ROOT)), + "iteration": next_iteration(DEFAULT_ITERATION_ROOT), + "policy": "mcts", + "games": 1000, + "workers": 8, + "seed": int(time.time()) % 1000000, + "mctsSims": 80, + "mctsCpuct": 1.5, + "depth": 4, + "width": 32, + "timeMs": 1000, + "epochs": 8, + "policyEpochs": 8, + "benchmarkGames": 100, + "arenaGames": 100, + "maxTurns": 80, + "maxDecisions": 1000, + } + + +HTML = r""" + + + + + Firebat Self-Play + + + +
+

Firebat Self-Play

+
Idle
+
+
+
+

Run

+
+ + + + + + + + + + + + + + + +
+
+ + +
+
+
+
+

Progress

+ No run +
+
+
Games
0/0
+
Policy Rows
0
+
Avg Move Ms
0.0
+
Nodes
0
+
+
+
+
+ 

+      

+    

+    

+      
Charts
0 checkpoints

+      

+        
Strength

+        
Side Win Rates

+        
Training

+        
Throughput

+      

+    

+  

+  
+
+
+"""
+
+
+class Handler(BaseHTTPRequestHandler):
+    def log_message(self, format, *args):
+        return
+
+    def _json(self, value, status=200):
+        payload = json.dumps(value).encode("utf-8")
+        self.send_response(status)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(payload)))
+        self.end_headers()
+        self.wfile.write(payload)
+
+    def _text(self, value, status=200, content_type="text/plain; charset=utf-8"):
+        payload = value.encode("utf-8")
+        self.send_response(status)
+        self.send_header("Content-Type", content_type)
+        self.send_header("Content-Length", str(len(payload)))
+        self.end_headers()
+        self.wfile.write(payload)
+
+    def do_GET(self):
+        path = urlparse(self.path).path
+        if path == "/":
+            self._text(HTML, content_type="text/html; charset=utf-8")
+        elif path == "/api/status":
+            self._json(STATE.status())
+        else:
+            self._text("not found", status=404)
+
+    def do_POST(self):
+        path = urlparse(self.path).path
+        length = int(self.headers.get("Content-Length", "0"))
+        body = self.rfile.read(length).decode("utf-8") if length else "{}"
+        try:
+            data = json.loads(body or "{}")
+        except json.JSONDecodeError:
+            data = {}
+
+        if path == "/api/start":
+            try:
+                self._json(STATE.start(data))
+            except Exception as exc:
+                self._text(str(exc), status=409)
+        elif path == "/api/stop":
+            self._json({"stopped": STATE.stop()})
+        else:
+            self._text("not found", status=404)
+
+
+def daemonize(args):
+    DASH_ROOT.mkdir(parents=True, exist_ok=True)
+    child_args = [arg for arg in sys.argv[1:] if arg != "--daemon"]
+    log_path = DASH_ROOT / "server.log"
+    with open(log_path, "a", buffering=1, encoding="utf-8") as log:
+        process = subprocess.Popen(
+            [sys.executable, str(Path(__file__).resolve()), *child_args],
+            cwd=str(REPO_ROOT),
+            stdout=log,
+            stderr=subprocess.STDOUT,
+            start_new_session=True,
+            text=True,
+        )
+    print(json.dumps({"pid": process.pid, "url": f"http://127.0.0.1:{args.port}", "log": str(log_path)}))
+
+
+def serve(args):
+    server = ThreadingHTTPServer((args.host, args.port), Handler)
+
+    if args.start:
+        params = default_start_params()
+        params.update({
+            "outputDir": args.output_dir,
+            "iteration": args.iteration or next_iteration(resolve_path(args.output_dir, DEFAULT_ITERATION_ROOT)),
+            "policy": args.policy,
+            "games": args.games,
+            "workers": args.workers,
+            "seed": args.seed,
+            "mctsSims": args.mcts_sims,
+            "mctsCpuct": args.mcts_cpuct,
+            "depth": args.depth,
+            "width": args.width,
+            "timeMs": args.time_ms,
+            "epochs": args.epochs,
+            "policyEpochs": args.policy_epochs,
+            "benchmarkGames": args.benchmark_games,
+            "arenaGames": args.arena_games,
+        })
+        STATE.start(params)
+
+    print(f"Firebat dashboard listening on http://{args.host}:{args.port}", flush=True)
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        pass
+    finally:
+        STATE.stop()
+        server.server_close()
+
+
+def parse_args():
+    defaults = default_start_params()
+    parser = argparse.ArgumentParser(description="Local dashboard for Firebat Naxx self-play.")
+    parser.add_argument("--host", default="127.0.0.1")
+    parser.add_argument("--port", type=int, default=8765)
+    parser.add_argument("--daemon", action="store_true")
+    parser.add_argument("--start", action="store_true")
+    parser.add_argument("--run-job")
+    parser.add_argument("--output-dir", default=defaults["outputDir"])
+    parser.add_argument("--iteration", type=int, default=0)
+    parser.add_argument("--policy", default=defaults["policy"])
+    parser.add_argument("--games", type=int, default=defaults["games"])
+    parser.add_argument("--workers", type=int, default=defaults["workers"])
+    parser.add_argument("--seed", type=int, default=defaults["seed"])
+    parser.add_argument("--mcts-sims", type=int, default=defaults["mctsSims"])
+    parser.add_argument("--mcts-cpuct", type=float, default=defaults["mctsCpuct"])
+    parser.add_argument("--depth", type=int, default=defaults["depth"])
+    parser.add_argument("--width", type=int, default=defaults["width"])
+    parser.add_argument("--time-ms", type=int, default=defaults["timeMs"])
+    parser.add_argument("--epochs", type=int, default=defaults["epochs"])
+    parser.add_argument("--policy-epochs", type=int, default=defaults["policyEpochs"])
+    parser.add_argument("--benchmark-games", type=int, default=defaults["benchmarkGames"])
+    parser.add_argument("--arena-games", type=int, default=defaults["arenaGames"])
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    if args.run_job:
+        run_parallel_job(args.run_job)
+    elif args.daemon:
+        daemonize(args)
+    else:
+        serve(args)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/train_firebat_value.py b/tools/train_firebat_value.py
new file mode 100644
index 000000000..adfe6c4b1
--- /dev/null
+++ b/tools/train_firebat_value.py
@@ -0,0 +1,489 @@
+#!/usr/bin/env python3
+"""Train bootstrap linear policy/value models from Firebat rollout JSONL.
+
+The model is intentionally small: value = tanh(w dot x).  It is meant as a
+cheap initialization target before search-generated self-play data exists.
+Policy heads are also linear softmax models over action features when policy
+rows are present.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import random
+from collections import Counter, defaultdict
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Sequence, Tuple
+
+
+DRUID_DECK = [
+    "FB_Champs_EX1_169",
+    "FB_Champs_EX1_169",
+    "CS2_013",
+    "CS2_013",
+    "EX1_154",
+    "EX1_154",
+    "CS2_011",
+    "CS2_011",
+    "CS2_012",
+    "CS2_012",
+    "FB_Champs_EX1_166",
+    "FB_Champs_EX1_166",
+    "EX1_571",
+    "FB_Champs_EX1_165",
+    "FB_Champs_EX1_165",
+    "FB_Champs_NEW1_008",
+    "FB_Champs_NEW1_008",
+    "EX1_573",
+    "EX1_058",
+    "FP1_005",
+    "FP1_005",
+    "FB_Champs_EX1_005",
+    "CS2_182",
+    "CS2_182",
+    "EX1_002",
+    "EX1_284",
+    "FP1_030",
+    "FP1_008",
+    "FP1_008",
+    "EX1_110",
+]
+
+ZOO_DECK = [
+    "EX1_319",
+    "EX1_319",
+    "EX1_316",
+    "EX1_308",
+    "EX1_308",
+    "CS2_065",
+    "CS2_065",
+    "EX1_304",
+    "EX1_310",
+    "EX1_310",
+    "FB_Champs_CS2_188",
+    "FB_Champs_CS2_188",
+    "FB_Champs_EX1_029",
+    "FB_Champs_EX1_029",
+    "FB_Champs_FP1_028",
+    "FB_Champs_FP1_028",
+    "EX1_162",
+    "EX1_162",
+    "FP1_002",
+    "FP1_002",
+    "FB_Champs_NEW1_019",
+    "FB_Champs_NEW1_019",
+    "FP1_007",
+    "FP1_007",
+    "FB_Champs_EX1_556",
+    "FB_Champs_EX1_556",
+    "EX1_046",
+    "EX1_093",
+    "EX1_093",
+    "FP1_030",
+]
+
+CARD_IDS = sorted(set(DRUID_DECK + ZOO_DECK + ["GAME_005"]))
+SCHEMA = "firebat_naxx_linear_value_v1"
+
+
+def build_feature_schema() -> List[str]:
+    features = [
+        "bias",
+        "turn_norm",
+        "current_player_is_viewer",
+        "viewer_is_combo_druid",
+        "viewer_is_zoo_warlock",
+    ]
+    for prefix in ("viewer", "opponent"):
+        features.extend(
+            [
+                f"{prefix}_hero_health_norm",
+                f"{prefix}_hero_armor_norm",
+                f"{prefix}_base_mana_norm",
+                f"{prefix}_remaining_mana_norm",
+                f"{prefix}_hand_count_norm",
+                f"{prefix}_deck_count_norm",
+                f"{prefix}_board_count_norm",
+                f"{prefix}_board_attack_norm",
+                f"{prefix}_board_health_norm",
+                f"{prefix}_taunt_count_norm",
+            ]
+        )
+    features.extend(
+        [
+            "hero_health_delta_norm",
+            "board_attack_delta_norm",
+            "board_health_delta_norm",
+        ]
+    )
+    for prefix in ("viewer", "opponent"):
+        for zone in ("hand", "deck", "board", "graveyard"):
+            for card_id in CARD_IDS:
+                features.append(f"{prefix}_{zone}_{card_id}")
+    return features
+
+
+FEATURE_SCHEMA = build_feature_schema()
+
+
+def board_attack(controller: Dict[str, Any]) -> int:
+    return sum(max(0, int(minion.get("attack", 0))) for minion in controller.get("board", []))
+
+
+def board_health(controller: Dict[str, Any]) -> int:
+    return sum(max(0, int(minion.get("health", 0))) for minion in controller.get("board", []))
+
+
+def add_card_counts(features: Dict[str, float], prefix: str, card_ids: Iterable[str], normalizer: float = 2.0) -> None:
+    for card_id, count in Counter(card_ids).items():
+        features[f"{prefix}_{card_id}"] = count / normalizer
+
+
+def add_controller_features(features: Dict[str, float], prefix: str, controller: Dict[str, Any]) -> None:
+    board = controller.get("board", [])
+    features[f"{prefix}_hero_health_norm"] = float(controller.get("heroHealth", 0)) / 30.0
+    features[f"{prefix}_hero_armor_norm"] = float(controller.get("heroArmor", 0)) / 30.0
+    features[f"{prefix}_base_mana_norm"] = float(controller.get("baseMana", 0)) / 10.0
+    features[f"{prefix}_remaining_mana_norm"] = float(controller.get("remainingMana", 0)) / 10.0
+    features[f"{prefix}_hand_count_norm"] = float(controller.get("handCount", 0)) / 10.0
+    features[f"{prefix}_deck_count_norm"] = float(controller.get("deckCount", 0)) / 30.0
+    features[f"{prefix}_board_count_norm"] = len(board) / 7.0
+    features[f"{prefix}_board_attack_norm"] = board_attack(controller) / 30.0
+    features[f"{prefix}_board_health_norm"] = board_health(controller) / 60.0
+    features[f"{prefix}_taunt_count_norm"] = sum(1 for minion in board if minion.get("taunt")) / 7.0
+
+    add_card_counts(features, f"{prefix}_hand", controller.get("handIds", []))
+    add_card_counts(features, f"{prefix}_deck", controller.get("deckIds", []))
+    add_card_counts(features, f"{prefix}_board", (minion.get("id", "") for minion in board))
+    add_card_counts(features, f"{prefix}_graveyard", controller.get("graveyard", []))
+
+
+def featurize(observation: Dict[str, Any]) -> List[float]:
+    viewer = observation["viewer"]
+    opponent = observation["opponent"]
+    features: Dict[str, float] = {
+        "bias": 1.0,
+        "turn_norm": float(observation.get("turn", 0)) / 20.0,
+        "current_player_is_viewer": 1.0
+        if int(observation.get("currentPlayerId", -1)) == int(viewer.get("playerId", -2))
+        else 0.0,
+        "viewer_is_combo_druid": 1.0 if viewer.get("side") == "combo_druid" else 0.0,
+        "viewer_is_zoo_warlock": 1.0 if viewer.get("side") == "zoo_warlock" else 0.0,
+    }
+    add_controller_features(features, "viewer", viewer)
+    add_controller_features(features, "opponent", opponent)
+    features["hero_health_delta_norm"] = (
+        float(viewer.get("heroHealth", 0))
+        + float(viewer.get("heroArmor", 0))
+        - float(opponent.get("heroHealth", 0))
+        - float(opponent.get("heroArmor", 0))
+    ) / 30.0
+    features["board_attack_delta_norm"] = (board_attack(viewer) - board_attack(opponent)) / 30.0
+    features["board_health_delta_norm"] = (board_health(viewer) - board_health(opponent)) / 60.0
+    return [features.get(name, 0.0) for name in FEATURE_SCHEMA]
+
+
+def dot(weights: Sequence[float], values: Sequence[float]) -> float:
+    return sum(w * x for w, x in zip(weights, values))
+
+
+def train_model(
+    rows: List[Tuple[int, List[float], float]],
+    epochs: int,
+    learning_rate: float,
+    l2: float,
+    holdout_mod: int,
+    seed: int,
+) -> Tuple[List[float], Dict[str, float]]:
+    if not rows:
+        return [0.0] * len(FEATURE_SCHEMA), {
+            "trainRows": 0,
+            "validationRows": 0,
+            "trainMse": 0.0,
+            "validationMse": 0.0,
+        }
+
+    train = [row for row in rows if holdout_mod <= 1 or row[0] % holdout_mod != 0]
+    validation = [row for row in rows if holdout_mod > 1 and row[0] % holdout_mod == 0]
+    if not train:
+        train = rows[:]
+    if not validation:
+        validation = rows[:]
+
+    weights = [0.0] * len(FEATURE_SCHEMA)
+    rng = random.Random(seed)
+    for epoch in range(max(1, epochs)):
+        rng.shuffle(train)
+        lr = learning_rate / (1.0 + epoch * 0.05)
+        for _, x, y in train:
+            score = dot(weights, x)
+            pred = math.tanh(score)
+            grad_common = (pred - y) * (1.0 - pred * pred)
+            for i, value in enumerate(x):
+                if value == 0.0 and weights[i] == 0.0:
+                    continue
+                weights[i] -= lr * (grad_common * value + l2 * weights[i])
+
+    metrics = {
+        "trainRows": len(train),
+        "validationRows": len(validation),
+        "trainMse": mse(weights, train),
+        "validationMse": mse(weights, validation),
+    }
+    return weights, metrics
+
+
+def mse(weights: Sequence[float], rows: Sequence[Tuple[int, List[float], float]]) -> float:
+    if not rows:
+        return 0.0
+    total = 0.0
+    for _, x, y in rows:
+        err = math.tanh(dot(weights, x)) - y
+        total += err * err
+    return total / len(rows)
+
+
+def load_rows(paths: Sequence[Path]) -> Dict[Tuple[str, str], List[Tuple[int, List[float], float]]]:
+    rows: Dict[Tuple[str, str], List[Tuple[int, List[float], float]]] = defaultdict(list)
+    for path in paths:
+        with path.open("r", encoding="utf-8") as handle:
+            for line in handle:
+                if not line.strip():
+                    continue
+                row = json.loads(line)
+                kind = row.get("kind")
+                if kind not in ("legal_value", "full_info_value"):
+                    continue
+                side = row.get("side")
+                if side not in ("combo_druid", "zoo_warlock"):
+                    continue
+                value = float(row.get("value", 0.0))
+                game_seed = int(row.get("gameSeed", 0))
+                rows[(kind, side)].append((game_seed, featurize(row["observation"]), value))
+    return rows
+
+
+PolicyAction = Tuple[Dict[str, float], float]
+PolicyRow = Tuple[int, List[PolicyAction]]
+
+
+def load_policy_rows(paths: Sequence[Path]) -> Dict[str, List[PolicyRow]]:
+    rows: Dict[str, List[PolicyRow]] = defaultdict(list)
+    for path in paths:
+        with path.open("r", encoding="utf-8") as handle:
+            for line in handle:
+                if not line.strip():
+                    continue
+                row = json.loads(line)
+                if row.get("kind") != "policy":
+                    continue
+                side = row.get("side")
+                if side not in ("combo_druid", "zoo_warlock"):
+                    continue
+                actions: List[PolicyAction] = []
+                for action in row.get("legalActions", []):
+                    features = action.get("features", {})
+                    if not isinstance(features, dict):
+                        continue
+                    actions.append(({str(k): float(v) for k, v in features.items()}, float(action.get("target", 0.0))))
+                target_total = sum(max(0.0, target) for _, target in actions)
+                if actions and target_total > 0:
+                    normalized = [(features, max(0.0, target) / target_total) for features, target in actions]
+                    rows[side].append((int(row.get("gameSeed", 0)), normalized))
+    return rows
+
+
+def build_action_feature_schema(rows_by_side: Dict[str, List[PolicyRow]]) -> List[str]:
+    keys = set()
+    for rows in rows_by_side.values():
+        for _, actions in rows:
+            for features, _ in actions:
+                keys.update(features.keys())
+    if not keys:
+        return []
+    ordered = ["bias"] if "bias" in keys else []
+    ordered.extend(sorted(key for key in keys if key != "bias"))
+    return ordered
+
+
+def policy_vector(features: Dict[str, float], schema: Sequence[str]) -> List[float]:
+    return [features.get(name, 0.0) for name in schema]
+
+
+def softmax(logits: Sequence[float]) -> List[float]:
+    if not logits:
+        return []
+    shift = max(logits)
+    values = [math.exp(max(-40.0, value - shift)) for value in logits]
+    total = sum(values)
+    if total <= 0:
+        return [1.0 / len(values)] * len(values)
+    return [value / total for value in values]
+
+
+def train_policy_model(
+	rows: List[PolicyRow],
+	action_schema: Sequence[str],
+	epochs: int,
+	learning_rate: float,
+	l2: float,
+	holdout_mod: int,
+	seed: int,
+) -> Tuple[List[float], Dict[str, float]]:
+    if not rows or not action_schema:
+        return [0.0] * len(action_schema), {
+            "trainRows": 0,
+            "validationRows": 0,
+            "trainAccuracy": 0.0,
+            "validationAccuracy": 0.0,
+            "trainNll": 0.0,
+            "validationNll": 0.0,
+        }
+
+    train = [row for row in rows if holdout_mod <= 1 or row[0] % holdout_mod != 0]
+    validation = [row for row in rows if holdout_mod > 1 and row[0] % holdout_mod == 0]
+    if not train:
+        train = rows[:]
+    if not validation:
+        validation = rows[:]
+
+    weights = [0.0] * len(action_schema)
+    rng = random.Random(seed)
+    for epoch in range(max(1, epochs)):
+        rng.shuffle(train)
+        lr = learning_rate / (1.0 + epoch * 0.05)
+        for _, actions in train:
+            xs = [policy_vector(features, action_schema) for features, _ in actions]
+            targets = [target for _, target in actions]
+            logits = [dot(weights, x) for x in xs]
+            probs = softmax(logits)
+            for x, prob, target in zip(xs, probs, targets):
+                grad_common = prob - target
+                for i, value in enumerate(x):
+                    if value == 0.0 and weights[i] == 0.0:
+                        continue
+                    weights[i] -= lr * (grad_common * value + l2 * weights[i])
+
+    metrics = {
+        "trainRows": len(train),
+        "validationRows": len(validation),
+        "trainAccuracy": policy_accuracy(weights, train, action_schema),
+        "validationAccuracy": policy_accuracy(weights, validation, action_schema),
+        "trainNll": policy_nll(weights, train, action_schema),
+        "validationNll": policy_nll(weights, validation, action_schema),
+    }
+    return weights, metrics
+
+
+def policy_accuracy(weights: Sequence[float], rows: Sequence[PolicyRow], action_schema: Sequence[str]) -> float:
+    if not rows:
+        return 0.0
+    correct = 0
+    for _, actions in rows:
+        xs = [policy_vector(features, action_schema) for features, _ in actions]
+        targets = [target for _, target in actions]
+        if not xs:
+            continue
+        predicted = max(range(len(xs)), key=lambda idx: dot(weights, xs[idx]))
+        target = max(range(len(targets)), key=lambda idx: targets[idx])
+        if predicted == target:
+            correct += 1
+    return correct / len(rows)
+
+
+def policy_nll(weights: Sequence[float], rows: Sequence[PolicyRow], action_schema: Sequence[str]) -> float:
+    if not rows:
+        return 0.0
+    total = 0.0
+    for _, actions in rows:
+        xs = [policy_vector(features, action_schema) for features, _ in actions]
+        targets = [target for _, target in actions]
+        probs = softmax([dot(weights, x) for x in xs])
+        for prob, target in zip(probs, targets):
+            if target > 0.0:
+                total -= math.log(max(prob, 1e-12)) * target
+    return total / len(rows)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("input", nargs="+", type=Path, help="Rollout JSONL file(s).")
+    parser.add_argument("--output", type=Path, default=Path("firebat-naxx-linear-value.json"), help="Output model JSON.")
+    parser.add_argument("--epochs", type=int, default=30)
+    parser.add_argument("--learning-rate", type=float, default=0.03)
+    parser.add_argument("--policy-epochs", type=int, default=None)
+    parser.add_argument("--policy-learning-rate", type=float, default=None)
+    parser.add_argument("--l2", type=float, default=0.0001)
+    parser.add_argument("--holdout-mod", type=int, default=5)
+    parser.add_argument("--seed", type=int, default=1)
+    args = parser.parse_args()
+
+    rows_by_model = load_rows(args.input)
+    policy_rows_by_side = load_policy_rows(args.input)
+    action_feature_schema = build_action_feature_schema(policy_rows_by_side)
+    models = []
+    for kind in ("legal_value", "full_info_value"):
+        for side in ("combo_druid", "zoo_warlock"):
+            weights, metrics = train_model(
+                rows_by_model.get((kind, side), []),
+                epochs=args.epochs,
+                learning_rate=args.learning_rate,
+                l2=args.l2,
+                holdout_mod=args.holdout_mod,
+                seed=args.seed + len(models) * 97,
+            )
+            model = {
+                "kind": kind,
+                "side": side,
+                "weights": weights,
+            }
+            model.update(metrics)
+            models.append(model)
+            print(
+                f"{kind}:{side} train={metrics['trainRows']} validation={metrics['validationRows']} "
+                f"trainMse={metrics['trainMse']:.4f} validationMse={metrics['validationMse']:.4f}"
+            )
+
+    policy_models = []
+    for side in ("combo_druid", "zoo_warlock"):
+        weights, metrics = train_policy_model(
+            policy_rows_by_side.get(side, []),
+            action_feature_schema,
+            epochs=args.policy_epochs if args.policy_epochs is not None else args.epochs,
+            learning_rate=args.policy_learning_rate if args.policy_learning_rate is not None else args.learning_rate,
+            l2=args.l2,
+            holdout_mod=args.holdout_mod,
+            seed=args.seed + 1009 + len(policy_models) * 97,
+        )
+        model = {
+            "kind": "legal_policy",
+            "side": side,
+            "weights": weights,
+        }
+        model.update(metrics)
+        policy_models.append(model)
+        print(
+            f"legal_policy:{side} train={metrics['trainRows']} validation={metrics['validationRows']} "
+            f"trainAcc={metrics['trainAccuracy']:.3f} validationAcc={metrics['validationAccuracy']:.3f} "
+            f"trainNll={metrics['trainNll']:.3f} validationNll={metrics['validationNll']:.3f}"
+        )
+
+    payload = {
+        "schema": SCHEMA,
+        "featureSchema": FEATURE_SCHEMA,
+        "actionFeatureSchema": action_feature_schema,
+        "models": models,
+        "policyModels": policy_models,
+    }
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    with args.output.open("w", encoding="utf-8") as handle:
+        json.dump(payload, handle, indent=2, sort_keys=True)
+        handle.write("\n")
+    print(f"wrote {args.output}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())