Upload 5 files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+policy_9x9_p1 filter=lfs diff=lfs merge=lfs -text
+policy_9x9_p2 filter=lfs diff=lfs merge=lfs -text

app_9x9.py

@@ -0,0 +1,122 @@
+import streamlit as st
+import numpy as np
+from object_9x9 import Player, HumanPlayer
+
+# Create RL bot player and human player
+p1 = Player("9x9_p1")
+human_player = HumanPlayer("p2")
+
+def handle_click(i, j):
+    if (i, j) not in check_available_moves(extra=True):
+        st.session_state.warning = True
+    elif not st.session_state.winner:
+        st.session_state.warning = False
+        st.session_state.board[i, j] = st.session_state.player
+        winner = check_win(st.session_state.board)
+        if winner:
+            st.session_state.winner = winner
+            if st.session_state.opponent == 'Computer':
+                # Give reward to the RL bot and update its policy
+                if winner == 'X':
+                    p1.feedReward(1)
+                elif winner == 'O':
+                    p1.feedReward(0)
+                else:
+                    p1.feedReward(0.1)
+                
+                # Save the RL bot's policy
+                p1.savePolicy()
+        
+        # Toggle the player's turn
+        st.session_state.player = "O" if st.session_state.player == "X" else "X"
+
+def init(post_init=False):
+    if not post_init:
+        st.session_state.win = {'X': 0, 'O': 0}
+        st.session_state.opponent = 'Computer'  # Initialize the opponent attribute
+    st.session_state.board = np.full((9, 9), '.', dtype=str)
+    st.session_state.player = 'X'
+    st.session_state.warning = False
+    st.session_state.winner = None
+    st.session_state.over = False
+
+def check_available_moves(extra=False) -> list:
+    raw_moves = [row for col in st.session_state.board.tolist() for row in col]
+    num_moves = [i for i, spot in enumerate(raw_moves) if spot == '.']
+    if extra:
+        return [(i // 9, i % 9) for i in num_moves]
+    return num_moves
+
+def check_win(board):
+    # Check for a winner by checking rows, columns, and diagonals for three linked points
+    for i in range(9):
+        for j in range(9):
+            if board[i, j] != '.':
+                symbol = board[i, j]
+                # Check horizontally
+                if j + 2 < 9 and board[i, j + 1] == board[i, j + 2] == symbol:
+                    return symbol
+                # Check vertically
+                if i + 2 < 9 and board[i + 1, j] == board[i + 2, j] == symbol:
+                    return symbol
+                # Check diagonally (top-left to bottom-right)
+                if i + 2 < 9 and j + 2 < 9 and board[i + 1, j + 1] == board[i + 2, j + 2] == symbol:
+                    return symbol
+                # Check diagonally (top-right to bottom-left)
+                if i + 2 < 9 and j - 2 >= 0 and board[i + 1, j - 1] == board[i + 2, j - 2] == symbol:
+                    return symbol
+    return None
+
+def computer_player():
+    moves = check_available_moves(extra=True)
+    if moves:
+        # Use p1 to choose the action
+        positions = check_available_moves(extra=True)
+        p1_action = p1.chooseAction(positions, st.session_state.board, -1)
+        
+        # Check if the chosen action is valid and make the move
+        if p1_action in moves:
+            i, j = p1_action
+            handle_click(i, j)
+
+def main():
+    st.write(
+        """
+        # ❎🅾️ Tic Tac Toe (9x9)
+        """
+    )
+
+    if "board" not in st.session_state:
+        init()
+
+    reset, score, player = st.columns([0.5, 0.6, 1])
+    reset.button('New game', on_click=init, args=(True,))
+
+    for i, row in enumerate(st.session_state.board):
+        cols = st.columns([5] + [1] * 9 + [5])
+        for j, field in enumerate(row):
+            if st.session_state.player == 'X' and not st.session_state.winner:
+                # Only allow human player to make a move if it's their turn and the game is not over
+                cols[j + 1].button(
+                    field,
+                    key=f"{i}-{j}",
+                    on_click=lambda i=i, j=j: handle_click(i, j)  # Use lambda to pass arguments
+                )
+            else:
+                # If it's not the human player's turn or the game is over, let the bot play
+                computer_player()
+
+    if st.session_state.winner:
+        st.success(f"Congrats! {st.session_state.winner} won the game! 🎈")
+    elif not check_available_moves():
+        st.info(f"It's a tie 📍")
+
+    score.button(f'❌{st.session_state.win["X"]} 🆚 {st.session_state.win["O"]}⭕')
+    player.button(
+        f'{"❌" if st.session_state.player == "X" else "⭕"}\'s turn'
+        if not st.session_state.winner
+        else f'🏁 Game finished'
+    )
+
+if __name__ == '__main__':
+    main()

object_9x9.py

@@ -0,0 +1,278 @@
+import numpy as np
+import pickle
+
+BOARD_ROWS = 9
+BOARD_COLS = 9
+
+
+class State:
+    def __init__(self, p1, p2):
+        self.board = np.zeros((BOARD_ROWS, BOARD_COLS))
+        self.p1 = p1
+        self.p2 = p2
+        self.isEnd = False
+        self.boardHash = None
+        # init p1 plays first
+        self.playerSymbol = 1
+
+    # get unique hash of current board state
+    def getHash(self):
+        self.boardHash = str(self.board.reshape(BOARD_COLS * BOARD_ROWS))
+        return self.boardHash
+
+    def winner(self):
+        # row
+        for i in range(BOARD_ROWS):
+            if sum(self.board[i, :]) == 3:
+                self.isEnd = True
+                return 1
+            if sum(self.board[i, :]) == -3:
+                self.isEnd = True
+                return -1
+        # col
+        for i in range(BOARD_COLS):
+            if sum(self.board[:, i]) == 3:
+                self.isEnd = True
+                return 1
+            if sum(self.board[:, i]) == -3:
+                self.isEnd = True
+                return -1
+        # diagonal
+        diag_sum1 = sum([self.board[i, i] for i in range(BOARD_COLS)])
+        diag_sum2 = sum([self.board[i, BOARD_COLS - i - 1] for i in range(BOARD_COLS)])
+        diag_sum = max(abs(diag_sum1), abs(diag_sum2))
+        if diag_sum == 3:
+            self.isEnd = True
+            if diag_sum1 == 3 or diag_sum2 == 3:
+                return 1
+            else:
+                return -1
+
+        # tie
+        # no available positions
+        if len(self.availablePositions()) == 0:
+            self.isEnd = True
+            return 0
+        # not end
+        self.isEnd = False
+        return None
+
+    def availablePositions(self):
+        positions = []
+        for i in range(BOARD_ROWS):
+            for j in range(BOARD_COLS):
+                if self.board[i, j] == 0:
+                    positions.append((i, j))  # need to be tuple
+        return positions
+
+    def updateState(self, position):
+        self.board[position] = self.playerSymbol
+        # switch to another player
+        self.playerSymbol = -1 if self.playerSymbol == 1 else 1
+
+    # only when game ends
+    def giveReward(self):
+        result = self.winner()
+        # backpropagate reward
+        if result == 1:
+            self.p1.feedReward(1)
+            self.p2.feedReward(0)
+        elif result == -1:
+            self.p1.feedReward(0)
+            self.p2.feedReward(1)
+        else:
+            self.p1.feedReward(0.1)
+            self.p2.feedReward(0.5)
+
+    # board reset
+    def reset(self):
+        self.board = np.zeros((BOARD_ROWS, BOARD_COLS))
+        self.boardHash = None
+        self.isEnd = False
+        self.playerSymbol = 1
+
+    def playwithbot(self, rounds=100):
+        for i in range(rounds):
+            if i % 1000 == 0:
+                print("Rounds {}".format(i))
+            while not self.isEnd:
+                # Player 1
+                positions = self.availablePositions()
+                p1_action = self.p1.chooseAction(positions, self.board, self.playerSymbol)
+                # take action and upate board state
+                self.updateState(p1_action)
+                board_hash = self.getHash()
+                self.p1.addState(board_hash)
+                # check board status if it is end
+
+                win = self.winner()
+                if win is not None:
+                    # self.showBoard()
+                    # ended with p1 either win or draw
+                    self.giveReward()
+                    self.p1.reset()
+                    self.p2.reset()
+                    self.reset()
+                    break
+
+                else:
+                    # Player 2
+                    positions = self.availablePositions()
+                    p2_action = self.p2.chooseAction(positions, self.board, self.playerSymbol)
+                    self.updateState(p2_action)
+                    board_hash = self.getHash()
+                    self.p2.addState(board_hash)
+
+                    win = self.winner()
+                    if win is not None:
+                        # self.showBoard()
+                        # ended with p2 either win or draw
+                        self.giveReward()
+                        self.p1.reset()
+                        self.p2.reset()
+                        self.reset()
+                        break
+
+    # play with human
+    def playwithhuman(self):
+        while not self.isEnd:
+            # Player 1
+            positions = self.availablePositions()
+            p1_action = self.p1.chooseAction(positions, self.board, self.playerSymbol)
+            # take action and upate board state
+            self.updateState(p1_action)
+            self.showBoard()
+            # check board status if it is end
+            win = self.winner()
+            if win is not None:
+                if win == 1:
+                    print(self.p1.name, "wins!")
+                else:
+                    print("tie!")
+                self.reset()
+                break
+
+            else:
+                # Player 2
+                positions = self.availablePositions()
+                p2_action = self.p2.chooseAction(positions)
+
+                self.updateState(p2_action)
+                self.showBoard()
+                win = self.winner()
+                if win is not None:
+                    if win == -1:
+                        print(self.p2.name, "wins!")
+                    else:
+                        print("tie!")
+                    self.reset()
+                    break
+
+    # def showBoard(self):
+    #     # p1: x  p2: o
+    #     for i in range(0, BOARD_ROWS):
+    #         print('-------------')
+    #         out = '| '
+    #         for j in range(0, BOARD_COLS):
+    #             if self.board[i, j] == 1:
+    #                 token = 'x'
+    #             if self.board[i, j] == -1:
+    #                 token = 'o'
+    #             if self.board[i, j] == 0:
+    #                 token = ' '
+    #             out += token + ' | '
+    #         print(out)
+    #     print('-------------')
+
+
+class Player:
+    def __init__(self, name, exp_rate=0.3):
+        self.name = name
+        self.states = []  # record all positions taken
+        self.lr = 0.2
+        self.exp_rate = exp_rate
+        self.decay_gamma = 0.9
+        self.states_value = {}  # state -> value
+        self.loadPolicy('policy_' + str(self.name))  # Load the pre-trained policy
+
+    def getHash(self, board):
+        boardHash = str(board.reshape(BOARD_COLS * BOARD_ROWS))
+        return boardHash
+
+    def chooseAction(self, positions, current_board, symbol):
+        if np.random.uniform(0, 1) <= self.exp_rate:
+            # take random action
+            idx = np.random.choice(len(positions))
+            action = positions[idx]
+        else:
+            value_max = -999
+            for p in positions:
+                next_board = current_board.copy()
+                next_board[p] = symbol
+                next_boardHash = self.getHash(next_board)
+                value = 0 if self.states_value.get(next_boardHash) is None else self.states_value.get(next_boardHash)
+                # print("value", value)
+                if value >= value_max:
+                    value_max = value
+                    action = p
+        # print("{} takes action {}".format(self.name, action))
+        return action
+
+    # append a hash state
+    def addState(self, state):
+        self.states.append(state)
+
+    # at the end of game, backpropagate and update states value
+    def feedReward(self, reward):
+        for st in reversed(self.states):
+            if self.states_value.get(st) is None:
+                self.states_value[st] = 0
+            self.states_value[st] += self.lr * (self.decay_gamma * reward - self.states_value[st])
+            reward = self.states_value[st]
+
+    def reset(self):
+        self.states = []
+
+    def savePolicy(self):
+        fw = open('policy_9x9_' + str(self.name), 'wb')
+        pickle.dump(self.states_value, fw)
+        fw.close()
+
+    def loadPolicy(self, file):
+        fr = open(file, 'rb')
+        self.states_value = pickle.load(fr)
+        fr.close()
+
+
+class HumanPlayer:
+    def __init__(self, name):
+        self.name = name
+
+    def chooseAction(self, positions):
+        pass
+
+    # append a hash state
+    def addState(self, state):
+        pass
+
+    # at the end of game, backpropagate and update states value
+    def feedReward(self, reward):
+        pass
+
+    def reset(self):
+        pass
+
+
+if __name__ == "__main__":
+    # training
+    p1 = Player("p1")
+    p2 = Player("p2")
+
+    st = State(p1, p2)
+    print("training...")
+    st.playwithbot(2000)
+
+    p1.savePolicy()
+    p2.savePolicy()
+
+

policy_9x9_p1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df6ea3d6afd12a02c2249ed25c2e259b379d3c7e27127fc45b311f9716625c7c
+size 34402854

policy_9x9_p2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9bc0ddefb84074489fa5f9c6fda365f3b85e48f090624f15d427d820bb7d3bf0
+size 37261567

requirements.txt

@@ -0,0 +1,2 @@
+numpy
+streamlit