Merge pull request aimacode#471 from jsuyash1514/mcts3e

Monte Carlo Tree Search - AIMA3e

Author: samagra14

aima-core/src/main/java/aima/core/environment/tictactoe/TicTacToeState.java

@@ -19,11 +19,22 @@ public class TicTacToeState implements Cloneable {
 	public static final String X = "X";
 	public static final String EMPTY = "-";
 	//
-	private String[] board = new String[] { EMPTY, EMPTY, EMPTY, EMPTY, EMPTY,
-			EMPTY, EMPTY, EMPTY, EMPTY };
+	private String[] board;
 
-	private String playerToMove = X;
+	private String playerToMove;
 	private double utility = -1; // 1: win for X, 0: win for O, 0.5: draw
+	
+	public TicTacToeState(){
+		this.board = new String[] { EMPTY, EMPTY, EMPTY, EMPTY, EMPTY, EMPTY, EMPTY, EMPTY, EMPTY };
+		playerToMove = X;
+	}
+	
+	public TicTacToeState(String[] board, String playerToMove){
+		this.board = board;
+		this.playerToMove = (Objects.equals(playerToMove, X) ? O : X);
+		analyzeUtility();
+		this.playerToMove = playerToMove;
+	}
 
 	public String getPlayerToMove() {
 		return playerToMove;

aima-core/src/main/java/aima/core/environment/twoply/TwoPlyGame.java

@@ -0,0 +1,76 @@
+package aima.core.environment.twoply;
+
+import aima.core.environment.map.MoveToAction;
+import aima.core.search.adversarial.Game;
+
+import java.util.List;
+
+public class TwoPlyGame implements Game<TwoPlyGameState, MoveToAction, String> {
+	
+	@Override
+	public TwoPlyGameState getInitialState() {
+		return new TwoPlyGameState("A");
+	}
+	
+	@Override
+	public String[] getPlayers() {
+		return new String[]{"MAX", "MIN"};
+	}
+	
+	@Override
+	public String getPlayer(TwoPlyGameState state) {
+		switch (state.getLocation()) {
+			case "B":
+			case "C":
+			case "D":
+				return "MIN";
+			default:
+				return "MAX";
+		}
+	}
+	
+	
+	@Override
+	public List<MoveToAction> getActions(TwoPlyGameState state) {
+		return new TwoPlyGameTree().getActions(state);
+	}
+	
+	@Override
+	public TwoPlyGameState getResult(TwoPlyGameState state, MoveToAction action) {
+		return new TwoPlyGameState(action.getToLocation());
+	}
+	
+	@Override
+	public boolean isTerminal(TwoPlyGameState state) {
+		return (state.getLocation().charAt(0) > 'D');
+	}
+	
+	@Override
+	public double getUtility(TwoPlyGameState state, String player) {
+		switch (state.getLocation()) {
+			// B
+			case "E":
+				return 3;
+			case "F":
+				return 12;
+			case "G":
+				return 8;
+			// C
+			case "H":
+				return 2;
+			case "I":
+				return 4;
+			case "J":
+				return 6;
+			// D
+			case "K":
+				return 14;
+			case "L":
+				return 5;
+			case "M":
+				return 2;
+			default:
+				throw new IllegalArgumentException("State " + state.getLocation() + " unexpected.");
+		}
+	}
+}

aima-core/src/main/java/aima/core/environment/twoply/TwoPlyGameState.java

@@ -0,0 +1,31 @@
+package aima.core.environment.twoply;
+
+public class TwoPlyGameState {
+	private String location;
+	
+	public TwoPlyGameState(String location) {
+		this.location = location;
+	}
+	
+	public String getLocation() {
+		return location;
+	}
+	
+	@Override
+	public boolean equals(Object obj) {
+		if (obj instanceof TwoPlyGameState) {
+			return this.getLocation().equals(((TwoPlyGameState) obj).getLocation());
+		}
+		return super.equals(obj);
+	}
+	
+	@Override
+	public int hashCode() {
+		return getLocation().hashCode();
+	}
+	
+	@Override
+	public String toString() {
+		return location;
+	}
+}

aima-core/src/main/java/aima/core/environment/twoply/TwoPlyGameTree.java

@@ -0,0 +1,42 @@
+package aima.core.environment.twoply;
+
+import aima.core.environment.map.ExtendableMap;
+import aima.core.environment.map.Map;
+import aima.core.environment.map.MoveToAction;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class TwoPlyGameTree {
+	Map aima3eFig5_2;
+	
+	public TwoPlyGameTree() {
+		aima3eFig5_2 = new ExtendableMap() {
+			{
+				addUnidirectionalLink("A", "B", 1.0);
+				addUnidirectionalLink("A", "C", 1.0);
+				addUnidirectionalLink("A", "D", 1.0);
+				addUnidirectionalLink("B", "E", 1.0);
+				addUnidirectionalLink("B", "F", 1.0);
+				addUnidirectionalLink("B", "G", 1.0);
+				addUnidirectionalLink("C", "H", 1.0);
+				addUnidirectionalLink("C", "I", 1.0);
+				addUnidirectionalLink("C", "J", 1.0);
+				addUnidirectionalLink("D", "K", 1.0);
+				addUnidirectionalLink("D", "L", 1.0);
+				addUnidirectionalLink("D", "M", 1.0);
+			}
+		};
+	}
+	
+	public List<MoveToAction> getActions(TwoPlyGameState state) {
+		List<MoveToAction> possibleActions = new ArrayList<>();
+		List<String> nextPossibleLocations = aima3eFig5_2.getPossibleNextLocations(state.getLocation());
+		for (String nextLocation : nextPossibleLocations) {
+			MoveToAction action = new MoveToAction(nextLocation);
+			possibleActions.add(action);
+		}
+		return possibleActions;
+	}
+	
+}

aima-core/src/main/java/aima/core/search/adversarial/MonteCarloTreeSearch.java

@@ -0,0 +1,146 @@
+package aima.core.search.adversarial;
+
+import aima.core.search.framework.GameTree;
+import aima.core.search.framework.Metrics;
+import aima.core.search.framework.Node;
+import aima.core.search.framework.NodeFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+/**
+ * Artificial Intelligence A Modern Approach (4th Edition): page ???.<br>
+ *
+ * <pre>
+ * <code>
+ * function MONTE-CARLO-TREE-SEARCH(state) returns an action
+ *   tree &larr; NODE(state)
+ *   while TIME-REMAINING() do
+ *   	leaf &larr; SELECT(tree)
+ *   	child &larr; EXPAND(leaf)
+ *   	result &larr; SIMULATE(child)
+ *   	BACKPROPAGATE(result, child)
+ *   return the move in ACTIONS(state) whose node has highest number of playouts
+ * </code>
+ * </pre>
+ *
+ * Figure ?.? The Monte Carlo tree search algorithm. A game tree, tree, is initialized, and
+ * then we repeat the cycle of SELECT / EXPAND / SIMULATE/ BACKPROPAGATE until we run  out
+ * of time, and return the move that led to the node with the highest number of playouts.
+ *
+ *
+ * @author Suyash Jain
+ *
+ * @param <S>
+ *            Type which is used for states in the game.
+ * @param <A>
+ *            Type which is used for actions in the game.
+ * @param <P>
+ *            Type which is used for players in the game.
+ */
+
+public class MonteCarloTreeSearch<S, A, P> implements AdversarialSearch<S, A> {
+	private int iterations = 0;
+	private Game<S, A, P> game;
+	private GameTree<S, A> tree;
+	
+	public MonteCarloTreeSearch(Game<S, A, P> game, int iterations) {
+		this.game = game;
+		this.iterations = iterations;
+		tree = new GameTree<>();
+	}
+	
+	@Override
+	public A makeDecision(S state) {
+		// tree <-- NODE(state)
+		tree.addRoot(state);
+		// while TIME-REMAINING() do
+		while (iterations != 0) {
+			// leaf <-- SELECT(tree)
+			Node<S, A> leaf = select(tree);
+			// child <-- EXPAND(leaf)
+			Node<S, A> child = expand(leaf);
+			// result <-- SIMULATE(child)
+			// result = true if player of root node wins
+			boolean result = simulate(child);
+			// BACKPROPAGATE(result, child)
+			backpropagate(result, child);
+			// repeat the four steps for set number of iterations
+			--iterations;
+		}
+		// return the move in ACTIONS(state) whose node has highest number of playouts
+		return bestAction(tree.getRoot());
+	}
+	
+	private Node<S, A> select(GameTree gameTree) {
+		Node<S, A> node = gameTree.getRoot();
+		while (!game.isTerminal(node.getState()) && isNodeFullyExpanded(node)) {
+			node = gameTree.getChildWithMaxUCT(node);
+		}
+		return node;
+	}
+	
+	private Node<S, A> expand(Node<S, A> leaf) {
+		if (game.isTerminal(leaf.getState())) return leaf;
+		else {
+			Node<S, A> child = randomlySelectUnvisitedChild(leaf);
+			return child;
+		}
+	}
+	
+	private boolean simulate(Node<S, A> node) {
+		while (!game.isTerminal(node.getState())) {
+			Random rand = new Random();
+			A a = game.getActions(node.getState()).get(rand.nextInt(game.getActions(node.getState()).size()));
+			S result = game.getResult(node.getState(), a);
+			NodeFactory nodeFactory = new NodeFactory();
+			node = nodeFactory.createNode(result);
+		}
+		if (game.getUtility(node.getState(), game.getPlayer(tree.getRoot().getState())) > 0) return true;
+		else return false;
+	}
+	
+	private void backpropagate(boolean result, Node<S, A> node) {
+		tree.updateStats(result, node);
+		if (tree.getParent(node) != null) backpropagate(result, tree.getParent(node));
+	}
+	
+	private A bestAction(Node<S, A> root) {
+		Node<S, A> bestChild = tree.getChildWithMaxPlayouts(root);
+		for (A a : game.getActions(root.getState())) {
+			S result = game.getResult(root.getState(), a);
+			if (result.equals(bestChild.getState())) return a;
+		}
+		return null;
+	}
+	
+	private boolean isNodeFullyExpanded(Node<S, A> node) {
+		List<S> visitedChildren = tree.getVisitedChildren(node);
+		for (A a : game.getActions(node.getState())) {
+			S result = game.getResult(node.getState(), a);
+			if (!visitedChildren.contains(result)) {
+				return false;
+			}
+		}
+		return true;
+	}
+	
+	
+	private Node<S, A> randomlySelectUnvisitedChild(Node<S, A> node) {
+		List<S> unvisitedChildren = new ArrayList<>();
+		List<S> visitedChildren = tree.getVisitedChildren(node);
+		for (A a : game.getActions(node.getState())) {
+			S result = game.getResult(node.getState(), a);
+			if (!visitedChildren.contains(result)) unvisitedChildren.add(result);
+		}
+		Random rand = new Random();
+		Node<S, A> newChild = tree.addChild(node, unvisitedChildren.get(rand.nextInt(unvisitedChildren.size())));
+		return newChild;
+	}
+	
+	@Override
+	public Metrics getMetrics() {
+		return null;
+	}
+}