codeShare commited on
Commit
529a803
β€’
1 Parent(s): eabe046

Upload sd_token_similarity_calculator.ipynb

Browse files
Files changed (1) hide show
  1. sd_token_similarity_calculator.ipynb +124 -6
sd_token_similarity_calculator.ipynb CHANGED
@@ -25,6 +25,15 @@
25
  "id": "L7JTcbOdBPfh"
26
  }
27
  },
 
 
 
 
 
 
 
 
 
28
  {
29
  "cell_type": "code",
30
  "source": [
@@ -122,10 +131,31 @@
122
  "!git clone https://huggingface.co/datasets/codeShare/text-to-image-prompts\n"
123
  ],
124
  "metadata": {
125
- "id": "rUXQ73IbonHY"
 
 
 
 
126
  },
127
- "execution_count": null,
128
- "outputs": []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
129
  },
130
  {
131
  "cell_type": "code",
@@ -238,10 +268,98 @@
238
  "NUM_VOCAB_ITEMS = nA\n"
239
  ],
240
  "metadata": {
241
- "id": "ZMG4CThUAmwW"
 
 
 
 
 
242
  },
243
- "execution_count": null,
244
- "outputs": []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
245
  },
246
  {
247
  "cell_type": "code",
 
25
  "id": "L7JTcbOdBPfh"
26
  }
27
  },
28
+ {
29
+ "cell_type": "markdown",
30
+ "source": [
31
+ "THIS IS AN OLD VERSION. UP TO DATE VERSION CAN BE FOUND HERE : https://huggingface.co/datasets/codeShare/text-to-image-prompts/tree/main/Google%20Colab%20Notebooks"
32
+ ],
33
+ "metadata": {
34
+ "id": "W0T0dlDuCPTN"
35
+ }
36
+ },
37
  {
38
  "cell_type": "code",
39
  "source": [
 
131
  "!git clone https://huggingface.co/datasets/codeShare/text-to-image-prompts\n"
132
  ],
133
  "metadata": {
134
+ "id": "rUXQ73IbonHY",
135
+ "outputId": "5ba68c03-2c3e-4f47-c307-712ff431f7ec",
136
+ "colab": {
137
+ "base_uri": "https://localhost:8080/"
138
+ }
139
  },
140
+ "execution_count": 1,
141
+ "outputs": [
142
+ {
143
+ "output_type": "stream",
144
+ "name": "stdout",
145
+ "text": [
146
+ "/content\n",
147
+ "Cloning into 'text-to-image-prompts'...\n",
148
+ "remote: Enumerating objects: 2372, done.\u001b[K\n",
149
+ "remote: Counting objects: 100% (4/4), done.\u001b[K\n",
150
+ "remote: Compressing objects: 100% (4/4), done.\u001b[K\n",
151
+ "remote: Total 2372 (delta 0), reused 0 (delta 0), pack-reused 2368 (from 1)\u001b[K\n",
152
+ "Receiving objects: 100% (2372/2372), 20.75 MiB | 12.08 MiB/s, done.\n",
153
+ "Resolving deltas: 100% (417/417), done.\n",
154
+ "Updating files: 100% (1301/1301), done.\n",
155
+ "Filtering content: 100% (578/578), 2.21 GiB | 36.41 MiB/s, done.\n"
156
+ ]
157
+ }
158
+ ]
159
  },
160
  {
161
  "cell_type": "code",
 
268
  "NUM_VOCAB_ITEMS = nA\n"
269
  ],
270
  "metadata": {
271
+ "id": "ZMG4CThUAmwW",
272
+ "outputId": "40d41f92-d461-4517-caa9-94951abccdca",
273
+ "colab": {
274
+ "base_uri": "https://localhost:8080/",
275
+ "height": 1000
276
+ }
277
  },
278
+ "execution_count": 2,
279
+ "outputs": [
280
+ {
281
+ "output_type": "stream",
282
+ "name": "stdout",
283
+ "text": [
284
+ "reading πŸŽ€ fusion-t2i-danbooru-tags-1.json....\n",
285
+ "/content/text-to-image-prompts/danbooru/text\n",
286
+ "/content/text-to-image-prompts/danbooru/text_encodings\n",
287
+ "reading πŸŽ€ fusion-t2i-danbooru-tags-5.json....\n",
288
+ "/content/text-to-image-prompts/danbooru/text\n",
289
+ "/content/text-to-image-prompts/danbooru/text_encodings\n",
290
+ "reading πŸŽ€ fusion-t2i-danbooru-tags-9.json....\n",
291
+ "/content/text-to-image-prompts/danbooru/text\n",
292
+ "/content/text-to-image-prompts/danbooru/text_encodings\n",
293
+ "reading πŸŽ€ fusion-t2i-danbooru-tags-18.json....\n",
294
+ "/content/text-to-image-prompts/danbooru/text\n",
295
+ "/content/text-to-image-prompts/danbooru/text_encodings\n",
296
+ "reading πŸŽ€ fusion-t2i-danbooru-tags-17.json....\n",
297
+ "/content/text-to-image-prompts/danbooru/text\n",
298
+ "/content/text-to-image-prompts/danbooru/text_encodings\n",
299
+ "reading πŸŽ€ fusion-t2i-danbooru-tags-13.json....\n",
300
+ "/content/text-to-image-prompts/danbooru/text\n",
301
+ "/content/text-to-image-prompts/danbooru/text_encodings\n",
302
+ "reading πŸŽ€ fusion-t2i-danbooru-tags-2.json....\n",
303
+ "/content/text-to-image-prompts/danbooru/text\n",
304
+ "/content/text-to-image-prompts/danbooru/text_encodings\n",
305
+ "reading πŸŽ€ fusion-t2i-danbooru-tags-12.json....\n",
306
+ "/content/text-to-image-prompts/danbooru/text\n",
307
+ "/content/text-to-image-prompts/danbooru/text_encodings\n",
308
+ "reading πŸŽ€ fusion-t2i-danbooru-tags-6.json....\n",
309
+ "/content/text-to-image-prompts/danbooru/text\n",
310
+ "/content/text-to-image-prompts/danbooru/text_encodings\n",
311
+ "reading πŸŽ€ fusion-t2i-danbooru-tags-11.json....\n",
312
+ "/content/text-to-image-prompts/danbooru/text\n",
313
+ "/content/text-to-image-prompts/danbooru/text_encodings\n",
314
+ "reading πŸŽ€ fusion-t2i-danbooru-tags-4.json....\n",
315
+ "/content/text-to-image-prompts/danbooru/text\n",
316
+ "/content/text-to-image-prompts/danbooru/text_encodings\n",
317
+ "reading πŸŽ€ fusion-t2i-danbooru-tags-3.json....\n",
318
+ "/content/text-to-image-prompts/danbooru/text\n",
319
+ "/content/text-to-image-prompts/danbooru/text_encodings\n",
320
+ "reading πŸŽ€ fusion-t2i-danbooru-tags-15.json....\n",
321
+ "/content/text-to-image-prompts/danbooru/text\n",
322
+ "/content/text-to-image-prompts/danbooru/text_encodings\n",
323
+ "reading πŸŽ€ fusion-t2i-danbooru-tags-8.json....\n",
324
+ "/content/text-to-image-prompts/danbooru/text\n",
325
+ "/content/text-to-image-prompts/danbooru/text_encodings\n",
326
+ "reading πŸŽ€ fusion-t2i-danbooru-tags-10.json....\n",
327
+ "/content/text-to-image-prompts/danbooru/text\n",
328
+ "/content/text-to-image-prompts/danbooru/text_encodings\n",
329
+ "reading πŸŽ€ fusion-t2i-danbooru-tags-20.json....\n",
330
+ "/content/text-to-image-prompts/danbooru/text\n",
331
+ "/content/text-to-image-prompts/danbooru/text_encodings\n",
332
+ "reading πŸŽ€ fusion-t2i-danbooru-tags-19.json....\n",
333
+ "/content/text-to-image-prompts/danbooru/text\n",
334
+ "/content/text-to-image-prompts/danbooru/text_encodings\n",
335
+ "reading πŸŽ€ fusion-t2i-danbooru-tags-21.json....\n",
336
+ "/content/text-to-image-prompts/danbooru/text\n",
337
+ "/content/text-to-image-prompts/danbooru/text_encodings\n",
338
+ "reading πŸŽ€ fusion-t2i-danbooru-tags-16.json....\n",
339
+ "/content/text-to-image-prompts/danbooru/text\n",
340
+ "/content/text-to-image-prompts/danbooru/text_encodings\n",
341
+ "reading πŸŽ€ fusion-t2i-danbooru-tags-14.json....\n",
342
+ "/content/text-to-image-prompts/danbooru/text\n",
343
+ "/content/text-to-image-prompts/danbooru/text_encodings\n",
344
+ "reading πŸŽ€ fusion-t2i-danbooru-tags-7.json....\n",
345
+ "/content/text-to-image-prompts/danbooru/text\n",
346
+ "/content/text-to-image-prompts/danbooru/text_encodings\n"
347
+ ]
348
+ },
349
+ {
350
+ "output_type": "error",
351
+ "ename": "FileNotFoundError",
352
+ "evalue": "[Errno 2] No such file or directory: '/content/text-to-image-prompts/tokens/suffix/common/text'",
353
+ "traceback": [
354
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
355
+ "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)",
356
+ "\u001b[0;32m<ipython-input-2-490cc35e84d5>\u001b[0m in \u001b[0;36m<cell line: 83>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 85\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mitem\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'common'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'average'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'rare'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'weird'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'exotic'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 86\u001b[0m \u001b[0murl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtmp\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 87\u001b[0;31m \u001b[0mprompts\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mtext_encodings\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnA\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mappend_from_url\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprompts\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mtext_encodings\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnA\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 88\u001b[0m \u001b[0;31m#------#\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 89\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
357
+ "\u001b[0;32m<ipython-input-1-8be73a8413a2>\u001b[0m in \u001b[0;36mappend_from_url\u001b[0;34m(dictA, tensA, nA, url, separator)\u001b[0m\n\u001b[1;32m 76\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 77\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mappend_from_url\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdictA\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensA\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mnA\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mseparator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 78\u001b[0;31m \u001b[0mdictB\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mtensB\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnB\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetPrompts\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mseparator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 79\u001b[0m \u001b[0mdictAB\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdictA\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 80\u001b[0m \u001b[0mtensAB\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtensA\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
358
+ "\u001b[0;32m<ipython-input-1-8be73a8413a2>\u001b[0m in \u001b[0;36mgetPrompts\u001b[0;34m(_path, separator)\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[0m_text_encodings\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[0;31m#-----#\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 29\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mfilename\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'{path}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 31\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'reading {filename}....'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
359
+ "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/content/text-to-image-prompts/tokens/suffix/common/text'"
360
+ ]
361
+ }
362
+ ]
363
  },
364
  {
365
  "cell_type": "code",