Hugging Face's logo

Spaces:
crpatel
/
Gujarati-BPE-Tokenizer
Sleeping

App Files Community
crpatel commited on
Commit
8846920
·
1 Parent(s): 1b7925a

bootstrap theme update

Browse files
Files changed (2) hide show
  1. app.py +1 -2
  2. static/index.html +39 -31
app.py CHANGED
@@ -37,6 +37,7 @@ async def read_root():
37
  @app.post("/encode")
38
  async def encode_text(request: EncodeRequest):
39
  """Encodes the input text and returns the tokens."""
 
40
  return {"encoded_tokens": tokenizer.encode(request.text)}
41
 
42
  @app.post("/decode")
@@ -45,7 +46,5 @@ async def decode_tokens(request: DecodeRequest):
45
  print(request.tokens)
46
  tokens = request.tokens.split(',')
47
  tokens = list(map(int, tokens))
48
- print(tokens, [type(token) for token in tokens])
49
- print(tokens)
50
  decoded_text = tokenizer.decode(tokens)
51
  return {"decoded_text": decoded_text}
 
37
  @app.post("/encode")
38
  async def encode_text(request: EncodeRequest):
39
  """Encodes the input text and returns the tokens."""
40
+ print("request.text: ", request.text)
41
  return {"encoded_tokens": tokenizer.encode(request.text)}
42
 
43
  @app.post("/decode")
 
46
  print(request.tokens)
47
  tokens = request.tokens.split(',')
48
  tokens = list(map(int, tokens))
 
 
49
  decoded_text = tokenizer.decode(tokens)
50
  return {"decoded_text": decoded_text}
static/index.html CHANGED
@@ -8,7 +8,7 @@
8
  <link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet">
9
  <style>
10
  .container {
11
- max-width: 800px;
12
  margin-top: 2rem;
13
  }
14
  .result-box {
@@ -60,41 +60,49 @@
60
  </head>
61
  <body class="bg-light">
62
  <div class="container">
63
- <h1 class="text-center mb-4">Basic Gujarati BPE Tokenizer</h1>
64
  <p class="text-center text-muted mb-4">5000 Vocabulary Size</p>
65
 
66
- <div class="card mb-4">
67
- <div class="card-header">
68
- <h2 class="h5 mb-0">Encode Text</h2>
69
- </div>
70
- <div class="card-body">
71
- <div class="mb-3">
72
- <input type="text" class="form-control mb-2" id="inputText" placeholder="Enter text to encode...">
73
- <button class="btn btn-primary" onclick="encode()">Encode</button>
74
- </div>
75
- <div class="result-box">
76
- <p class="mb-2"><strong>Encoded Tokens:</strong></p>
77
- <p class="mb-2"><span id="encodedTokens" class="text-break"></span></p>
78
- <p class="mb-2"><strong>Original Sentence:</strong></p>
79
- <p class="mb-0"><span id="originalSentence" class="text-break"></span></p>
 
 
 
 
 
80
  </div>
81
  </div>
82
- </div>
83
 
84
- <div class="card mb-4">
85
- <div class="card-header">
86
- <h2 class="h5 mb-0">Decode Tokens</h2>
87
- </div>
88
- <div class="card-body">
89
- <div class="mb-3">
90
- <input type="text" class="form-control mb-2" id="inputTokens" placeholder="Enter tokens to decode...">
91
- <button class="btn btn-primary" onclick="decode()">Decode</button>
92
- </div>
93
- <div class="result-box">
94
- <p class="mb-2"><strong>Decoded Text:</strong></p>
95
- <p class="mb-2"><span id="decodedText" class="text-break"></span></p>
96
- <p class="mb-2"><strong>Original Tokens Inputted:</strong></p>
97
- <p class="mb-0"><span id="inputTokensDisplay" class="text-break"></span></p>
 
 
 
 
98
  </div>
99
  </div>
100
  </div>
 
8
  <link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet">
9
  <style>
10
  .container {
11
+ max-width: 1200px;
12
  margin-top: 2rem;
13
  }
14
  .result-box {
 
60
  </head>
61
  <body class="bg-light">
62
  <div class="container">
63
+ <h1 class="text-center mb-4"> સરળ ગુજરાતી બી પી ઇ ટોકનાઇઝર (Basic Gujarati BPE Tokenizer) (૫૦૦૦શબ્દો)</h1>
64
  <p class="text-center text-muted mb-4">5000 Vocabulary Size</p>
65
 
66
+ <div class="row">
67
+ <!-- Encode Card -->
68
+ <div class="col-md-6 mb-4">
69
+ <div class="card h-100">
70
+ <div class="card-header">
71
+ <h2 class="h5 mb-0">Encode Text</h2>
72
+ </div>
73
+ <div class="card-body">
74
+ <div class="mb-3">
75
+ <input type="text" class="form-control mb-2" id="inputText" placeholder="Enter text to encode...">
76
+ <button class="btn btn-primary" onclick="encode()">Encode</button>
77
+ </div>
78
+ <div class="result-box">
79
+ <p class="mb-2"><strong>Encoded Tokens:</strong></p>
80
+ <p class="mb-2"><span id="encodedTokens" class="text-break"></span></p>
81
+ <p class="mb-2"><strong>Original Sentence:</strong></p>
82
+ <p class="mb-0"><span id="originalSentence" class="text-break"></span></p>
83
+ </div>
84
+ </div>
85
  </div>
86
  </div>
 
87
 
88
+ <!-- Decode Card -->
89
+ <div class="col-md-6 mb-4">
90
+ <div class="card h-100">
91
+ <div class="card-header">
92
+ <h2 class="h5 mb-0">Decode Tokens</h2>
93
+ </div>
94
+ <div class="card-body">
95
+ <div class="mb-3">
96
+ <input type="text" class="form-control mb-2" id="inputTokens" placeholder="Enter tokens to decode...">
97
+ <button class="btn btn-primary" onclick="decode()">Decode</button>
98
+ </div>
99
+ <div class="result-box">
100
+ <p class="mb-2"><strong>Decoded Text:</strong></p>
101
+ <p class="mb-2"><span id="decodedText" class="text-break"></span></p>
102
+ <p class="mb-2"><strong>Original Tokens Inputted:</strong></p>
103
+ <p class="mb-0"><span id="inputTokensDisplay" class="text-break"></span></p>
104
+ </div>
105
+ </div>
106
  </div>
107
  </div>
108
  </div>