shirastromer commited on
Commit
09a3e67
1 Parent(s): dba10d2

Training done

Browse files
Files changed (3) hide show
  1. added_tokens.json +11 -9
  2. tokenizer.json +26 -8
  3. tokenizer_config.json +24 -8
added_tokens.json CHANGED
@@ -1,10 +1,11 @@
1
  {
2
  "</s_date>": 57532,
3
  "</s_discount>": 57546,
4
- "</s_item_key>": 57548,
5
- "</s_item_name>": 57550,
6
- "</s_item_quantity>": 57554,
7
- "</s_item_value>": 57552,
 
8
  "</s_phone>": 57530,
9
  "</s_store_addr>": 57528,
10
  "</s_store_name>": 57526,
@@ -14,14 +15,15 @@
14
  "</s_time>": 57534,
15
  "</s_tips>": 57544,
16
  "</s_total>": 57542,
17
- "<s_cord-v2>": 57555,
18
  "<s_date>": 57531,
19
  "<s_discount>": 57545,
20
  "<s_iitcdip>": 57523,
21
- "<s_item_key>": 57547,
22
- "<s_item_name>": 57549,
23
- "<s_item_quantity>": 57553,
24
- "<s_item_value>": 57551,
 
25
  "<s_phone>": 57529,
26
  "<s_store_addr>": 57527,
27
  "<s_store_name>": 57525,
 
1
  {
2
  "</s_date>": 57532,
3
  "</s_discount>": 57546,
4
+ "</s_item_key>": 57550,
5
+ "</s_item_name>": 57552,
6
+ "</s_item_quantity>": 57556,
7
+ "</s_item_value>": 57554,
8
+ "</s_line_items>": 57548,
9
  "</s_phone>": 57530,
10
  "</s_store_addr>": 57528,
11
  "</s_store_name>": 57526,
 
15
  "</s_time>": 57534,
16
  "</s_tips>": 57544,
17
  "</s_total>": 57542,
18
+ "<s_cord-v2>": 57557,
19
  "<s_date>": 57531,
20
  "<s_discount>": 57545,
21
  "<s_iitcdip>": 57523,
22
+ "<s_item_key>": 57549,
23
+ "<s_item_name>": 57551,
24
+ "<s_item_quantity>": 57555,
25
+ "<s_item_value>": 57553,
26
+ "<s_line_items>": 57547,
27
  "<s_phone>": 57529,
28
  "<s_store_addr>": 57527,
29
  "<s_store_name>": 57525,
tokenizer.json CHANGED
@@ -289,7 +289,7 @@
289
  },
290
  {
291
  "id": 57547,
292
- "content": "<s_item_key>",
293
  "single_word": false,
294
  "lstrip": false,
295
  "rstrip": false,
@@ -298,7 +298,7 @@
298
  },
299
  {
300
  "id": 57548,
301
- "content": "</s_item_key>",
302
  "single_word": false,
303
  "lstrip": false,
304
  "rstrip": false,
@@ -307,7 +307,7 @@
307
  },
308
  {
309
  "id": 57549,
310
- "content": "<s_item_name>",
311
  "single_word": false,
312
  "lstrip": false,
313
  "rstrip": false,
@@ -316,7 +316,7 @@
316
  },
317
  {
318
  "id": 57550,
319
- "content": "</s_item_name>",
320
  "single_word": false,
321
  "lstrip": false,
322
  "rstrip": false,
@@ -325,7 +325,7 @@
325
  },
326
  {
327
  "id": 57551,
328
- "content": "<s_item_value>",
329
  "single_word": false,
330
  "lstrip": false,
331
  "rstrip": false,
@@ -334,7 +334,7 @@
334
  },
335
  {
336
  "id": 57552,
337
- "content": "</s_item_value>",
338
  "single_word": false,
339
  "lstrip": false,
340
  "rstrip": false,
@@ -343,7 +343,7 @@
343
  },
344
  {
345
  "id": 57553,
346
- "content": "<s_item_quantity>",
347
  "single_word": false,
348
  "lstrip": false,
349
  "rstrip": false,
@@ -352,7 +352,7 @@
352
  },
353
  {
354
  "id": 57554,
355
- "content": "</s_item_quantity>",
356
  "single_word": false,
357
  "lstrip": false,
358
  "rstrip": false,
@@ -361,6 +361,24 @@
361
  },
362
  {
363
  "id": 57555,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
364
  "content": "<s_cord-v2>",
365
  "single_word": false,
366
  "lstrip": false,
 
289
  },
290
  {
291
  "id": 57547,
292
+ "content": "<s_line_items>",
293
  "single_word": false,
294
  "lstrip": false,
295
  "rstrip": false,
 
298
  },
299
  {
300
  "id": 57548,
301
+ "content": "</s_line_items>",
302
  "single_word": false,
303
  "lstrip": false,
304
  "rstrip": false,
 
307
  },
308
  {
309
  "id": 57549,
310
+ "content": "<s_item_key>",
311
  "single_word": false,
312
  "lstrip": false,
313
  "rstrip": false,
 
316
  },
317
  {
318
  "id": 57550,
319
+ "content": "</s_item_key>",
320
  "single_word": false,
321
  "lstrip": false,
322
  "rstrip": false,
 
325
  },
326
  {
327
  "id": 57551,
328
+ "content": "<s_item_name>",
329
  "single_word": false,
330
  "lstrip": false,
331
  "rstrip": false,
 
334
  },
335
  {
336
  "id": 57552,
337
+ "content": "</s_item_name>",
338
  "single_word": false,
339
  "lstrip": false,
340
  "rstrip": false,
 
343
  },
344
  {
345
  "id": 57553,
346
+ "content": "<s_item_value>",
347
  "single_word": false,
348
  "lstrip": false,
349
  "rstrip": false,
 
352
  },
353
  {
354
  "id": 57554,
355
+ "content": "</s_item_value>",
356
  "single_word": false,
357
  "lstrip": false,
358
  "rstrip": false,
 
361
  },
362
  {
363
  "id": 57555,
364
+ "content": "<s_item_quantity>",
365
+ "single_word": false,
366
+ "lstrip": false,
367
+ "rstrip": false,
368
+ "normalized": true,
369
+ "special": false
370
+ },
371
+ {
372
+ "id": 57556,
373
+ "content": "</s_item_quantity>",
374
+ "single_word": false,
375
+ "lstrip": false,
376
+ "rstrip": false,
377
+ "normalized": true,
378
+ "special": false
379
+ },
380
+ {
381
+ "id": 57557,
382
  "content": "<s_cord-v2>",
383
  "single_word": false,
384
  "lstrip": false,
tokenizer_config.json CHANGED
@@ -241,7 +241,7 @@
241
  "special": false
242
  },
243
  "57547": {
244
- "content": "<s_item_key>",
245
  "lstrip": false,
246
  "normalized": true,
247
  "rstrip": false,
@@ -249,7 +249,7 @@
249
  "special": false
250
  },
251
  "57548": {
252
- "content": "</s_item_key>",
253
  "lstrip": false,
254
  "normalized": true,
255
  "rstrip": false,
@@ -257,7 +257,7 @@
257
  "special": false
258
  },
259
  "57549": {
260
- "content": "<s_item_name>",
261
  "lstrip": false,
262
  "normalized": true,
263
  "rstrip": false,
@@ -265,7 +265,7 @@
265
  "special": false
266
  },
267
  "57550": {
268
- "content": "</s_item_name>",
269
  "lstrip": false,
270
  "normalized": true,
271
  "rstrip": false,
@@ -273,7 +273,7 @@
273
  "special": false
274
  },
275
  "57551": {
276
- "content": "<s_item_value>",
277
  "lstrip": false,
278
  "normalized": true,
279
  "rstrip": false,
@@ -281,7 +281,7 @@
281
  "special": false
282
  },
283
  "57552": {
284
- "content": "</s_item_value>",
285
  "lstrip": false,
286
  "normalized": true,
287
  "rstrip": false,
@@ -289,7 +289,7 @@
289
  "special": false
290
  },
291
  "57553": {
292
- "content": "<s_item_quantity>",
293
  "lstrip": false,
294
  "normalized": true,
295
  "rstrip": false,
@@ -297,7 +297,7 @@
297
  "special": false
298
  },
299
  "57554": {
300
- "content": "</s_item_quantity>",
301
  "lstrip": false,
302
  "normalized": true,
303
  "rstrip": false,
@@ -305,6 +305,22 @@
305
  "special": false
306
  },
307
  "57555": {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
308
  "content": "<s_cord-v2>",
309
  "lstrip": false,
310
  "normalized": true,
 
241
  "special": false
242
  },
243
  "57547": {
244
+ "content": "<s_line_items>",
245
  "lstrip": false,
246
  "normalized": true,
247
  "rstrip": false,
 
249
  "special": false
250
  },
251
  "57548": {
252
+ "content": "</s_line_items>",
253
  "lstrip": false,
254
  "normalized": true,
255
  "rstrip": false,
 
257
  "special": false
258
  },
259
  "57549": {
260
+ "content": "<s_item_key>",
261
  "lstrip": false,
262
  "normalized": true,
263
  "rstrip": false,
 
265
  "special": false
266
  },
267
  "57550": {
268
+ "content": "</s_item_key>",
269
  "lstrip": false,
270
  "normalized": true,
271
  "rstrip": false,
 
273
  "special": false
274
  },
275
  "57551": {
276
+ "content": "<s_item_name>",
277
  "lstrip": false,
278
  "normalized": true,
279
  "rstrip": false,
 
281
  "special": false
282
  },
283
  "57552": {
284
+ "content": "</s_item_name>",
285
  "lstrip": false,
286
  "normalized": true,
287
  "rstrip": false,
 
289
  "special": false
290
  },
291
  "57553": {
292
+ "content": "<s_item_value>",
293
  "lstrip": false,
294
  "normalized": true,
295
  "rstrip": false,
 
297
  "special": false
298
  },
299
  "57554": {
300
+ "content": "</s_item_value>",
301
  "lstrip": false,
302
  "normalized": true,
303
  "rstrip": false,
 
305
  "special": false
306
  },
307
  "57555": {
308
+ "content": "<s_item_quantity>",
309
+ "lstrip": false,
310
+ "normalized": true,
311
+ "rstrip": false,
312
+ "single_word": false,
313
+ "special": false
314
+ },
315
+ "57556": {
316
+ "content": "</s_item_quantity>",
317
+ "lstrip": false,
318
+ "normalized": true,
319
+ "rstrip": false,
320
+ "single_word": false,
321
+ "special": false
322
+ },
323
+ "57557": {
324
  "content": "<s_cord-v2>",
325
  "lstrip": false,
326
  "normalized": true,