Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/demo_narrow.gif filter=lfs diff=lfs merge=lfs -text

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,17 @@
+<!-- Thank you for your contribution! -->
+
+<!-- Please add a reviewer to the assignee section when you create a PR. If you don't have the access to it, we will shortly find a reviewer and assign them to your PR. -->
+
+## Why are these changes needed?
+
+<!-- Please give a short summary of the change and the problem this solves. -->
+
+## Related issue number (if applicable)
+
+<!-- For example: "Closes #1234" -->
+
+## Checks
+
+- [ ] I've run `format.sh` to lint the changes in this PR.
+- [ ] I've included any doc changes needed.
+- [ ] I've made sure the relevant tests are passing (if applicable).

.github/workflows/python-package.yml

@@ -0,0 +1,30 @@
+name: Python package
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install -e '.[dev]'
+    - name: Run linter
+      run: |
+        pylint -d all -e E0602 ./fastchat/
+    - name: Check formatting
+      run: |
+        black --check .

.gitignore

@@ -0,0 +1,31 @@
+# Python
+__pycache__
+*.pyc
+*.egg-info
+dist
+.venv
+
+# Log
+*.log
+*.log.*
+*.json
+!playground/deepspeed_config_s2.json
+!playground/deepspeed_config_s3.json
+
+# Editor
+.idea
+*.swp
+
+# Other
+.DS_Store
+wandb
+output
+checkpoints_flant5_3b
+
+# Data
+*.pkl
+*.csv
+tests/state_of_the_union.txt
+
+# Build
+build

.pylintrc

@@ -0,0 +1,449 @@
+# This Pylint rcfile contains a best-effort configuration to uphold the
+# best-practices and style described in the Google Python style guide:
+#   https://google.github.io/styleguide/pyguide.html
+#
+# Its canonical open-source location is:
+#   https://google.github.io/styleguide/pylintrc
+
+[MASTER]
+
+# Files or directories to be skipped. They should be base names, not paths.
+ignore=third_party,ray_patches,providers
+
+# Files or directories matching the regex patterns are skipped. The regex
+# matches against base names, not paths.
+ignore-patterns=
+
+# Pickle collected data for later comparisons.
+persistent=no
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+# Use multiple processes to speed up Pylint.
+jobs=4
+
+# Allow loading of arbitrary C extensions. Extensions are imported into the
+# active Python interpreter and may run arbitrary code.
+unsafe-load-any-extension=no
+
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
+confidence=
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+#enable=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once).You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use"--disable=all --enable=classes
+# --disable=W"
+disable=abstract-method,
+        apply-builtin,
+        arguments-differ,
+        attribute-defined-outside-init,
+        backtick,
+        bad-option-value,
+        basestring-builtin,
+        buffer-builtin,
+        c-extension-no-member,
+        consider-using-enumerate,
+        cmp-builtin,
+        cmp-method,
+        coerce-builtin,
+        coerce-method,
+        delslice-method,
+        div-method,
+        duplicate-code,
+        eq-without-hash,
+        execfile-builtin,
+        file-builtin,
+        filter-builtin-not-iterating,
+        fixme,
+        getslice-method,
+        global-statement,
+        hex-method,
+        idiv-method,
+        implicit-str-concat-in-sequence,
+        import-error,
+        import-self,
+        import-star-module-level,
+        inconsistent-return-statements,
+        input-builtin,
+        intern-builtin,
+        invalid-str-codec,
+        locally-disabled,
+        logging-format-interpolation,  # FIXME(sky): make pass.
+        logging-fstring-interpolation,  # FIXME(sky): make pass.
+        long-builtin,
+        long-suffix,
+        map-builtin-not-iterating,
+        misplaced-comparison-constant,
+        missing-function-docstring,
+        metaclass-assignment,
+        next-method-called,
+        next-method-defined,
+        no-absolute-import,
+        no-else-break,
+        no-else-continue,
+        no-else-raise,
+        no-else-return,
+        no-init,  # added
+        no-member,
+        no-name-in-module,
+        no-self-use,
+        nonzero-method,
+        oct-method,
+        old-division,
+        old-ne-operator,
+        old-octal-literal,
+        old-raise-syntax,
+        parameter-unpacking,
+        print-statement,
+        raising-string,
+        range-builtin-not-iterating,
+        raw_input-builtin,
+        rdiv-method,
+        reduce-builtin,
+        relative-import,
+        reload-builtin,
+        round-builtin,
+        setslice-method,
+        signature-differs,
+        standarderror-builtin,
+        suppressed-message,
+        sys-max-int,
+        too-few-public-methods,
+        too-many-ancestors,
+        too-many-arguments,
+        too-many-boolean-expressions,
+        too-many-branches,
+        too-many-instance-attributes,
+        too-many-locals,
+        too-many-nested-blocks,
+        too-many-public-methods,
+        too-many-return-statements,
+        too-many-statements,
+        trailing-newlines,
+        unichr-builtin,
+        unicode-builtin,
+        unnecessary-pass,
+        unpacking-in-except,
+        useless-else-on-loop,
+        useless-object-inheritance,
+        useless-suppression,
+        using-cmp-argument,
+        wrong-import-order,
+        xrange-builtin,
+        zip-builtin-not-iterating,
+
+
+[REPORTS]
+
+# Set the output format. Available formats are text, parseable, colorized, msvs
+# (visual studio) and html. You can also give a reporter class, eg
+# mypackage.mymodule.MyReporterClass.
+output-format=text
+
+# Put messages in a separate file for each module / package specified on the
+# command line instead of printing them on stdout. Reports (if any) will be
+# written in a file name "pylint_global.[txt|html]". This option is deprecated
+# and it will be removed in Pylint 2.0.
+files-output=no
+
+# Tells whether to display a full report or only the messages
+reports=no
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note). You have access to the variables errors warning, statement which
+# respectively contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details
+#msg-template=
+
+
+[BASIC]
+
+# Good variable names which should always be accepted, separated by a comma
+good-names=main,_
+
+# Bad variable names which should always be refused, separated by a comma
+bad-names=
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Include a hint for the correct naming format with invalid-name
+include-naming-hint=no
+
+# List of decorators that produce properties, such as abc.abstractproperty. Add
+# to this list to register other decorators that produce valid properties.
+property-classes=abc.abstractproperty,cached_property.cached_property,cached_property.threaded_cached_property,cached_property.cached_property_with_ttl,cached_property.threaded_cached_property_with_ttl
+
+# Regular expression matching correct function names
+function-rgx=^(?:(?P<exempt>setUp|tearDown|setUpModule|tearDownModule)|(?P<camel_case>_?[A-Z][a-zA-Z0-9]*)|(?P<snake_case>_?[a-z][a-z0-9_]*))$
+
+# Regular expression matching correct variable names
+variable-rgx=^[a-z][a-z0-9_]*$
+
+# Regular expression matching correct constant names
+const-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$
+
+# Regular expression matching correct attribute names
+attr-rgx=^_{0,2}[a-z][a-z0-9_]*$
+
+# Regular expression matching correct argument names
+argument-rgx=^[a-z][a-z0-9_]*$
+
+# Regular expression matching correct class attribute names
+class-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$
+
+# Regular expression matching correct inline iteration names
+inlinevar-rgx=^[a-z][a-z0-9_]*$
+
+# Regular expression matching correct class names
+class-rgx=^_?[A-Z][a-zA-Z0-9]*$
+
+# Regular expression matching correct module names
+module-rgx=^(_?[a-z][a-z0-9_]*|__init__)$
+
+# Regular expression matching correct method names
+method-rgx=(?x)^(?:(?P<exempt>_[a-z0-9_]+__|runTest|setUp|tearDown|setUpTestCase|tearDownTestCase|setupSelf|tearDownClass|setUpClass|(test|assert)_*[A-Z0-9][a-zA-Z0-9_]*|next)|(?P<camel_case>_{0,2}[A-Z][a-zA-Z0-9_]*)|(?P<snake_case>_{0,2}[a-z][a-z0-9_]*))$
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=(__.*__|main|test.*|.*test|.*Test)$
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=10
+
+
+[TYPECHECK]
+
+# List of decorators that produce context managers, such as
+# contextlib.contextmanager. Add to this list to register other decorators that
+# produce valid context managers.
+contextmanager-decorators=contextlib.contextmanager,contextlib2.contextmanager
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis. It
+# supports qualified module names, as well as Unix pattern matching.
+ignored-modules=
+
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=optparse.Values,thread._local,_thread._local
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=
+
+
+[FORMAT]
+
+# Maximum number of characters on a single line.
+max-line-length=100
+
+# TODO(https://github.com/PyCQA/pylint/issues/3352): Direct pylint to exempt
+# lines made too long by directives to pytype.
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=(?x)(
+  ^\s*(\#\ )?<?https?://\S+>?$|
+  ^\s*(from\s+\S+\s+)?import\s+.+$)
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=yes
+
+# List of optional constructs for which whitespace checking is disabled. `dict-
+# separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
+# `trailing-comma` allows a space between comma and closing bracket: (a, ).
+# `empty-line` allows space-only lines.
+no-space-check=
+
+# Maximum number of lines in a module
+max-module-lines=99999
+
+# String used as indentation unit.  The internal Google style guide mandates 2
+# spaces.  Google's externaly-published style guide says 4, consistent with
+# PEP 8. Here we use 4 spaces.
+indent-string='    '
+
+# Number of spaces of indent required inside a hanging  or continued line.
+indent-after-paren=4
+
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=TODO
+
+
+[STRING]
+
+# This flag controls whether inconsistent-quotes generates a warning when the
+# character used as a quote delimiter is used inconsistently within a module.
+check-quote-consistency=yes
+
+
+[VARIABLES]
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# A regular expression matching the name of dummy variables (i.e. expectedly
+# not used).
+dummy-variables-rgx=^\*{0,2}(_$|unused_|dummy_)
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid to define new builtins when possible.
+additional-builtins=
+
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,_cb
+
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six,six.moves,past.builtins,future.builtins,functools
+
+
+[LOGGING]
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format
+logging-modules=logging,absl.logging,tensorflow.io.logging
+
+
+[SIMILARITIES]
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+# Ignore imports when computing similarities.
+ignore-imports=no
+
+
+[SPELLING]
+
+# Spelling dictionary name. Available dictionaries: none. To make it working
+# install python-enchant package.
+spelling-dict=
+
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
+
+# A path to a file that contains private dictionary; one word per line.
+spelling-private-dict-file=
+
+# Tells whether to store unknown words to indicated private dictionary in
+# --spelling-private-dict-file option instead of raising a message.
+spelling-store-unknown-words=no
+
+
+[IMPORTS]
+
+# Deprecated modules which should not be used, separated by a comma
+deprecated-modules=regsub,
+                   TERMIOS,
+                   Bastion,
+                   rexec,
+                   sets
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled)
+import-graph=
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled)
+ext-import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled)
+int-import-graph=
+
+# Force import order to recognize a module as part of the standard
+# compatibility libraries.
+known-standard-library=
+
+# Force import order to recognize a module as part of a third party library.
+known-third-party=enchant, absl
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+
+[CLASSES]
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,
+                      __new__,
+                      setUp
+
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,
+                  _fields,
+                  _replace,
+                  _source,
+                  _make
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls,
+                            class_
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=mcs
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "Exception"
+overgeneral-exceptions=StandardError,
+                       Exception,
+                       BaseException
+
+#######
+
+# https://github.com/edaniszewski/pylint-quotes#configuration
+string-quote=single
+triple-quote=double
+docstring-quote=double

2024-02-26-conv.json

@@ -0,0 +1,23 @@
+{"tstamp": 1708979292.1046, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1708979275.4427, "finish": 1708979292.1046, "state": {"template_name": "billa", "system_message": "", "roles": ["Human", "Assistant"], "messages": [["Human", "Hello "], ["Assistant", "Hello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello \n\nHello"]], "offset": 0, "conv_id": "0ea82788ded24b3e92860dc5b47febc0", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.50.130"}
+{"tstamp": 1708979428.0297, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1708979427.7589, "finish": 1708979428.0297, "state": {"template_name": "billa", "system_message": "", "roles": ["Human", "Assistant"], "messages": [["Human", "Hello how are you?"], ["Assistant", "Hello, how are you?"]], "offset": 0, "conv_id": "49f3fb8a82584c788578dda8e359e9d7", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.37.241"}
+{"tstamp": 1708979651.6512, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1708979651.3346, "finish": 1708979651.6512, "state": {"template_name": "billa", "system_message": "", "roles": ["Human", "Assistant"], "messages": [["Human", "Hello How are you?"], ["Assistant", "Hi, how are you doing?"]], "offset": 0, "conv_id": "1cb4f039ccde4e9abf3d237b47bc39d3", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.25.123"}
+{"tstamp": 1708979958.428, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1708979958.1858, "finish": 1708979958.428, "state": {"template_name": "one_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "Hello how are you?"], ["Assistant", "Hello, how are you?"]], "offset": 2, "conv_id": "4e4c5de905f64b64814562ce6cb7ca92", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.37.241"}
+{"tstamp": 1708980099.492, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1708980099.1622, "finish": 1708980099.492, "state": {"template_name": "one_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "Hello how are you?"], ["Assistant", "I am good thank you. How can I assist you today?"]], "offset": 2, "conv_id": "7a2f85c78ad244cfadbe214712847864", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.37.241"}
+{"tstamp": 1708980395.3758, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1708980394.8771, "finish": 1708980395.3758, "state": {"template_name": "one_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "Hello How are you?"], ["Assistant", "Hello! How can I help you?"]], "offset": 2, "conv_id": "dcd8306a9bb84462afa55336f94182da", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.37.241"}
+{"tstamp": 1708980429.3817, "type": "chat", "model": "MobiLlama-1B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1708980425.9794, "finish": 1708980429.3817, "state": {"template_name": "one_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "Generate a C code snippet that implements a function to calculate the Fibonacci sequence using recursion."], ["Assistant", "```c\n#include <stdio.h>\nint fib(int n);\nint fib(int n) {\n    if (n <= 1) {\n        return n;\n    }\n    else {\n        return fib(n-1) + fib(n-2);\n    }\n}\nint main() {\n    int n;\n    printf(\"Enter the value of n: \");\n    scanf(\"%d\", &n);\n    printf(\"The Fibonacci sequence is: \\n\");\n    for (int i = 0; i < n; i++) {\n        printf(\"%d \", fib(i));\n    }\n    return 0;\n}\n```\nThis C code implementation uses a recursive function `fib` to calculate the Fibonacci sequence. The function takes an integer `n` as input and returns the value of the `n`th Fibonacci"]], "offset": 2, "conv_id": "f250c5ef3c3d427582fe19e75839e236", "model_name": "MobiLlama-1B-Chat"}, "ip": "172.31.37.241"}
+{"tstamp": 1708980445.7322, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1708980442.5778, "finish": 1708980445.7322, "state": {"template_name": "one_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "Generate a C code snippet that implements a function to calculate the Fibonacci sequence using recursion."], ["Assistant", "```c\r\n#include <stdio.h>\r\n#include <stdlib.h>\r\n#include <math.h>\r\nint fibonacci(int n) {\r\n    if (n <= 1) {\r\n        return n;\r\n    }\r\n    else {\r\n        return fibonacci(n - 1) + fibonacci(n - 2);\r\n    }\r\n}\r\nint main() {\r\n    int n;\r\n    printf(\"Enter the first number: \");\r\n    scanf(\"%d\", &n);\r\n    printf(\"Enter the second number: \");\r\n    scanf(\"%d\", &n);\r\n    printf(\"The Fibonacci sequence is: %d %d %d\\n\", fibonacci(n), fibonacci(n-1), fibonacci(n-2));\r\n    return 0;\r\n}\r\n```\r\nThis function takes two parameters"]], "offset": 2, "conv_id": "e1f77bf7e30940db9addb35c3dd38294", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.12.31"}
+{"tstamp": 1708981892.4992, "type": "chat", "model": "MobiLlama-05B", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1708981891.9426, "finish": 1708981892.4992, "state": {"template_name": "one_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "I was walking in the garden when "], ["Assistant", "Then I saw your scissors, I cut them."]], "offset": 2, "conv_id": "573e0b8ebc9a419e814a0f24e4151e60", "model_name": "MobiLlama-05B"}, "ip": "172.31.50.130"}
+{"tstamp": 1708981911.5711, "type": "chat", "model": "MobiLlama-05B", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1708981911.2039, "finish": 1708981911.5711, "state": {"template_name": "one_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "How to write a python code to calculate factorial of a number"], ["Assistant", "How to write a python code to calculate factorial of a number"]], "offset": 2, "conv_id": "573e0b8ebc9a419e814a0f24e4151e60", "model_name": "MobiLlama-05B"}, "ip": "172.31.50.130"}
+{"tstamp": 1708981935.556, "type": "chat", "model": "MobiLlama-05B", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1708981935.3194, "finish": 1708981935.556, "state": {"template_name": "one_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "On a bright sunny day I was "], ["Assistant", "I'm a"]], "offset": 2, "conv_id": "573e0b8ebc9a419e814a0f24e4151e60", "model_name": "MobiLlama-05B"}, "ip": "172.31.12.31"}
+{"tstamp": 1708982111.1212, "type": "chat", "model": "MobiLlama-05B", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1708982110.8392, "finish": 1708982111.1212, "state": {"template_name": "zero_shot", "system_message": "", "roles": ["Human", "Assistant"], "messages": [["Human", "On a bright sunny day I was "], ["Assistant", "On the first day of an important"]], "offset": 0, "conv_id": "672719a556364698b8c1b5a9c7063529", "model_name": "MobiLlama-05B"}, "ip": "172.31.37.241"}
+{"tstamp": 1708982323.6584, "type": "chat", "model": "MobiLlama-05B", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1708982323.5031, "finish": 1708982323.6584, "state": {"template_name": "zero_shot", "system_message": "", "roles": ["Human", "Assistant"], "messages": [["Human", "On a bright sunny day I was "], ["Assistant", ""]], "offset": 0, "conv_id": "da84b42618f34b7bbaf4364a08c25dda", "model_name": "MobiLlama-05B"}, "ip": "172.31.37.241"}
+{"tstamp": 1708982336.4937, "type": "chat", "model": "MobiLlama-05B", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1708982336.3268, "finish": 1708982336.4937, "state": {"template_name": "zero_shot", "system_message": "", "roles": ["Human", "Assistant"], "messages": [["Human", "On a bright sunny day I was "], ["Assistant", ""]], "offset": 0, "conv_id": "da84b42618f34b7bbaf4364a08c25dda", "model_name": "MobiLlama-05B"}, "ip": "172.31.37.241"}
+{"tstamp": 1708982496.4846, "type": "chat", "model": "MobiLlama-05B", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1708982496.3086, "finish": 1708982496.4846, "state": {"template_name": "zero_shot", "system_message": "", "roles": ["Human", "Assistant"], "messages": [["Human", "On a bright sunny day I was "], ["Assistant", ""]], "offset": 0, "conv_id": "da84b42618f34b7bbaf4364a08c25dda", "model_name": "MobiLlama-05B"}, "ip": "172.31.25.123"}
+{"tstamp": 1708982505.7526, "type": "chat", "model": "MobiLlama-05B", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1708982505.6079, "finish": 1708982505.7526, "state": {"template_name": "zero_shot", "system_message": "", "roles": ["Human", "Assistant"], "messages": [["Human", "On a bright sunny day I was "], ["Assistant", ""]], "offset": 0, "conv_id": "da84b42618f34b7bbaf4364a08c25dda", "model_name": "MobiLlama-05B"}, "ip": "172.31.25.123"}
+{"tstamp": 1708983764.1363, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1708983763.3268, "finish": 1708983764.1363, "state": {"template_name": "one_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "Hi how are you?"], ["Assistant", "Hello! How can I help you today please \ud83d\ude0a"]], "offset": 2, "conv_id": "2325cc89da09435fa365959702fa2b56", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.37.241"}
+{"tstamp": 1708983783.8594, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1708983781.9502, "finish": 1708983783.8594, "state": {"template_name": "one_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "Write a python code to print a table of a number"], ["Assistant", "Sure! Here is one way you can do it using Python with NumPy arrays as data sources.\r\n```python\r\nimport numpy as np \r\nnumbers = [1024968753.,  // ASCII value for \"Happy\"\r\n            ---------------------// Countries in this alphabet are listed here too if desired (not shown)\r\n            ]) # Define array containing numbers from 'input_file', where file path given on command line argument or loaded into variable `numbs` similarily according specification provided by user below during execution time step"]], "offset": 2, "conv_id": "2325cc89da09435fa365959702fa2b56", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.37.241"}
+{"tstamp": 1708983818.622, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1708983816.9916, "finish": 1708983818.622, "state": {"template_name": "one_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "write a python code to print the factorial of a number"], ["Assistant", "here is one possible solution that could work for your problem without any additional libraries or dependencies on other platforms (Python 3 only):\r\n```python \r\ndef fact(n) :    if n <=0 else 'factor_of {}'.format((int)(fact))[1] + str(*list())[:-2][:-4].join() # this function uses `str` instead of regular expressions as it works with integers.\r\nprint(\"Fact value = \", end=\"\")"]], "offset": 2, "conv_id": "2325cc89da09435fa365959702fa2b56", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.12.31"}
+{"tstamp": 1708983829.2042, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1708983826.7386, "finish": 1708983829.2042, "state": {"template_name": "one_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "write a python code to print the factorial of a number"], ["Assistant", "Sure! Here is Python Code that can help you with your problem.\r\n```python\r\nfactor_factors = [1 ** i for i in range(2)] # initialize list containing factors from 0 upwards (starting at all numbers) \r\nprint(\"The Factorial of %d\"%num)\r\nfor num in factoars[::-1]:    if sum([f**i * f^k - k*sum()/j->p == j+len($elems)-4] > len('.')):           return True              break      else :                     continue   end        except StopIteration as e:`No such element found` or `None`:             raise ValueError(\"%r not positive integer\")"]], "offset": 2, "conv_id": "2325cc89da09435fa365959702fa2b56", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.12.31"}
+{"tstamp": 1708983854.4101, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1708983851.2397, "finish": 1708983854.4101, "state": {"template_name": "one_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "write a python code to calculate the factorial of a number"], ["Assistant", "Sure! Here is one way you can do it using Python as follows :  \r\n\r\n```python   # Assume we have already defined `factors` variable containing all possible factors up until n-1.\r\ndef fact(n):     return [factor for i in range(2**i) if (4*5 - sum([x * y + z] == s/d[j]) <= d[-3]*b))[:len(f)*6:-8].count((0).flatten())+[(a-.7)+(-9)-sum({_:[m:][\"identity\"]} < _ / g)[::-(g//k)]>= (\"Factoring\" not \"not algebraic\").strftime('%.%')]+pow(.|_.split(\".\")(int(_))))/(abs(__)!._!)*(reverse(*range(``.__list()))[_?<->](_) | reverse(\".\")>().index(!_(.)!(dig"]], "offset": 2, "conv_id": "2325cc89da09435fa365959702fa2b56", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.12.31"}
+{"tstamp": 1708983875.5177, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1708983872.355, "finish": 1708983875.5177, "state": {"template_name": "one_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "write a code in C language to find the factorial of a number"], ["Assistant", "Sure! Here is one possible implementation using recursion as shown above with some modifications based on your requirements.\r\n```c\r\nint fact(char c){ // define function call for finding faktor (factor) by character value x = 0; \r\n    if ((x == '1') && (!isdigit((unsigned char)(p))) ||!('\\n')){\r\n        printf(\"%d\\t\", p); exit();\r\n     }\r\n\tif (*--v!= '\\b'+'\\f');\r\n      else {\r\n         *++q++;;\r\n          while(*y > v-k+2&&*j >= k)\r\n                   j -= q - y + gpp(-4);\r\n           do{printf(\"-> %e%a:%u -> \", i++, n++){*(i++) += d[g];}\r\n               ++m,\r\n               5<=(39-(&putl()))?cout<<\"Shift left \":\""]], "offset": 2, "conv_id": "2325cc89da09435fa365959702fa2b56", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.12.31"}
+{"tstamp": 1708983904.2687, "type": "chat", "model": "MobiLlama-1B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1708983901.9497, "finish": 1708983904.2687, "state": {"template_name": "one_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "write a C language code to find the factorial of a number"], ["Assistant", "Here is one way you can implement this in Python using recursion: \n```python\ndef factorial(n):\n    if n <=1: # Base case for when n=0 or 1\n        return 1\n    else:   # Recursive call with n-1 as argument\n        result = n * factorial(n - 1)\n        print(\"The factorial of\", n,\"is\",result)\n    \nfactorial(5)        \nprint()      # This prints \"24\" since it returns 6 x 3! (which equals 72) multiplied by itself three times\n```"]], "offset": 2, "conv_id": "7db4fc8937364fd881dc9cf144dd1467", "model_name": "MobiLlama-1B-Chat"}, "ip": "172.31.50.130"}

2024-02-29-conv.json

@@ -0,0 +1,33 @@
+{"tstamp": 1709173113.8235, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709173113.5155, "finish": 1709173113.8235, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "who are you?"], ["Assistant", "I am an AI language model that helps people find information."]], "offset": 0, "conv_id": "74253ff2f9744483bdb9517b98bc58a3", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.12.31"}
+{"tstamp": 1709173839.3768, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709173838.7165, "finish": 1709173839.3768, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "What you know about Pakistan cricket?"], ["Assistant", "I'm an AI language model and don't have any knowledge about Pakistan cricket. However, I can provide you with some information about Pakistan cricket."]], "offset": 0, "conv_id": "3b298402f6474d0e872d1e7131b7e642", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.38.44"}
+{"tstamp": 1709173851.8231, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709173851.2365, "finish": 1709173851.8231, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "Yes sure."], ["Assistant", "As an AI language model, I don't have a physical body, but I can provide you with the information you are looking for."]], "offset": 0, "conv_id": "3b298402f6474d0e872d1e7131b7e642", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.38.44"}
+{"tstamp": 1709174172.9003, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709174172.6649, "finish": 1709174172.9003, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "Hii how are you "], ["Assistant", "Hi, how are you?"]], "offset": 0, "conv_id": "a1a8982247434c678f7f8e877dd0c20b", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.57.119"}
+{"tstamp": 1709174184.9852, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709174184.4893, "finish": 1709174184.9852, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "I'm fine "], ["Assistant", "Thanks for letting me know. I'm glad to hear that you're okay. Have you been feeling better lately?"]], "offset": 0, "conv_id": "a1a8982247434c678f7f8e877dd0c20b", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.57.119"}
+{"tstamp": 1709174203.8211, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709174202.7726, "finish": 1709174203.8211, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "What can you do for me?"], ["Assistant", "As an AI language model, I don't have a physical body, but I can provide you with information and help you with your queries. I can help you with your queries related to technology, finance, and any other topic you can think of. Please feel free to ask me anything you want."]], "offset": 0, "conv_id": "a1a8982247434c678f7f8e877dd0c20b", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.57.119"}
+{"tstamp": 1709176796.3134, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709176795.3603, "finish": 1709176796.3134, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "who are and which services you can provide?"], ["Assistant", "I can provide information and resources on various topics related to health and wellness, including nutrition, exercise, mental health, and more. I can also provide guidance on how to maintain a healthy lifestyle and support individuals who are struggling with their health."]], "offset": 0, "conv_id": "b158aa28a9ae4d58aaad2d0f5d61c577", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.57.119"}
+{"tstamp": 1709176839.6693, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709176833.48, "finish": 1709176839.6693, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "how can i use you in mobile apps? need detailed steps"], ["Assistant", "you can use you in mobile apps by following these steps:\n1. Install the app on your mobile device.\n2. Open the app and sign in with your account.\n3. Click on the \"Sign In\" button.\n4. Enter your username and password.\n5. Click on the \"Log In\" button.\n6. You will be redirected to the login page.\n7. Enter your username and password.\n8. Click on the \"Log In\" button.\n9. You will be redirected to the app dashboard.\n10. Click on the \"My Profile\" button.\n11. You will be redirected to your profile page.\n12. Click on the \"Edit Profile\" button.\n13. You will be redirected to your profile settings.\n14. Click on the \"Edit Profile\" button.\n15. You will be redirected to your profile picture.\n16. Click on the \"Upload Profile Picture\" button.\n17. You will be redirected to your profile picture.\n18. Click on the \"Save\" button.\n19. You will be redirected to your profile settings.\n20. Click on the \"Edit Profile\" button.\n21. You will be redirected to your profile settings.\n22. Click on the \"Edit Profile\" button.\n23. You will be redirected to your profile settings.\n24. Click on the \"Edit Profile\" button.\n25. You will be redirected to your profile settings.\n26. Click on the \"Edit Profile\" button.\n27. You will be redirected to your profile settings.\n28. Click on the \"Edit Profile\" button.\n29. You will be redirected to your profile settings.\n30. Click on the \"Edit Profile\" button."]], "offset": 0, "conv_id": "b158aa28a9ae4d58aaad2d0f5d61c577", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.57.119"}
+{"tstamp": 1709196117.2409, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709196116.9977, "finish": 1709196117.2409, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "\u3053\u3093\u306b\u3061\u306f"], ["Assistant", "\u3053\u3093\u306b\u3061\u306f"]], "offset": 0, "conv_id": "4dfac351421946bfb3c8e98d2cfef6a1", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.18.237"}
+{"tstamp": 1709196127.9141, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709196127.5827, "finish": 1709196127.9141, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "\u5143\u6c17\uff1f"], ["Assistant", "\u5143\u6c17\uff1f\n\n\u5143\u6c17\uff1f"]], "offset": 0, "conv_id": "4dfac351421946bfb3c8e98d2cfef6a1", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.18.237"}
+{"tstamp": 1709196140.8271, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709196140.5099, "finish": 1709196140.8271, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "\u4f55\u65452\u56de\uff1f"], ["Assistant", "\u56e0\u4e3a\u6211\u53eb\u666e\u901a\u4eba\u3002"]], "offset": 0, "conv_id": "4dfac351421946bfb3c8e98d2cfef6a1", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.18.237"}
+{"tstamp": 1709196162.1383, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709196161.8704, "finish": 1709196162.1383, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "\u65e5\u672c\u8a9e\u3067"], ["Assistant", "\u65e5\u672c\u8a9e\u3067"]], "offset": 0, "conv_id": "4dfac351421946bfb3c8e98d2cfef6a1", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.12.31"}
+{"tstamp": 1709196171.2674, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709196171.0041, "finish": 1709196171.2674, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "\u304a\u9858\u3044\u3057\u307e\u3059\u3002"], ["Assistant", "\u3054\u3081\u3093\u306d\u3002"]], "offset": 0, "conv_id": "4dfac351421946bfb3c8e98d2cfef6a1", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.12.31"}
+{"tstamp": 1709196180.1346, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709196179.9291, "finish": 1709196180.1346, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "\u3044\u3044\u3088"], ["Assistant", "\u3044\u3044\u3088"]], "offset": 0, "conv_id": "4dfac351421946bfb3c8e98d2cfef6a1", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.12.31"}
+{"tstamp": 1709196215.0201, "type": "chat", "model": "MobiLlama-1B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709196214.772, "finish": 1709196215.0201, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "\u7b11\u308f\u305b\u3066\u3002"], ["Assistant", "\u7b11\u308f\u305b\u3066\u3002"]], "offset": 0, "conv_id": "5af925ca8efb4d2e936734469a9058d6", "model_name": "MobiLlama-1B-Chat"}, "ip": "172.31.12.31"}
+{"tstamp": 1709196259.5968, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709196256.4629, "finish": 1709196259.5968, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "do you like japanese food?"], ["Assistant", "I am an AI language model and do not have emotions or preferences. However, I can provide you with some information about Japanese food. \nJapanese cuisine is known for its rich and diverse flavors. It is made up of a variety of ingredients and spices that are carefully selected to create unique and delicious dishes. Some of the most popular dishes in Japan include sushi, sashimi, tempura, ramen, and udon. \nJapanese cuisine is known for its use of high-quality ingredients and is often influenced by the seasons and local farmers markets. It is also known for its use of seasonal ingredients and the ability to adapt to different flavors and ingredients. \nOverall, Japanese food is a delicious and unique cuisine that is perfect for those who enjoy a variety of flavors and textures."]], "offset": 0, "conv_id": "383918313cd3485192d331cfe37ec1db", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.12.31"}
+{"tstamp": 1709203199.6921, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709203199.4168, "finish": 1709203199.6921, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "\u4f60\u597d"], ["Assistant", "Hi there! How can I assist you today?"]], "offset": 0, "conv_id": "e06b14bb931f40c29f4723b0eb9e3ecf", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.38.44"}
+{"tstamp": 1709203220.6204, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709203220.1293, "finish": 1709203220.6204, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "\u4f60\u61c2\u4e2d\u6587\u5417"], ["Assistant", "\u60a8\u597d\uff0c\u8bf7\u95ee\u4f60\u662f\u5426\u6709\u4efb\u4f55\u610f\u89c1\u548c\u5efa\u8bae\uff1f"]], "offset": 0, "conv_id": "e06b14bb931f40c29f4723b0eb9e3ecf", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.38.44"}
+{"tstamp": 1709203236.708, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709203236.3527, "finish": 1709203236.708, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "\u4f60\u53ef\u4ee5\u722c\u53d6\u7f51\u7ad9\u5185\u5bb9\u5417"], ["Assistant", "\u662f\u7684\uff0c\u6211\u53ef\u4ee5\u722c\u53d6\u7f51\u7ad9\u5185\u5bb9\u3002"]], "offset": 0, "conv_id": "e06b14bb931f40c29f4723b0eb9e3ecf", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.38.44"}
+{"tstamp": 1709203283.5876, "type": "chat", "model": "MobiLlama-1B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709203282.7635, "finish": 1709203283.5876, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "https://www.skroutz.gr/ \u4f60\u53ef\u4ee5\u5e2e\u6211\u722c\u53d6\u8fd9\u4e2a\u7f51\u7ad9\u7684\u5185\u5bb9\u5417"], ["Assistant", "\u53ef\u4ee5\uff0c\u6211\u4f1a\u5728\u8fd9\u4e2a\u7f51\u7ad9\u4e0a\u722c\u53d6\u4e00\u4e9b\u5185\u5bb9\uff0c\u5e76\u5c06\u5176\u6574\u7406\u6210\u4e00\u4e2a\u6587\u6863\uff0c\u6b22\u8fce\u60a8\u53c2\u4e0e\u3002"]], "offset": 0, "conv_id": "8450ba69ec064163b0997ff60744ca5e", "model_name": "MobiLlama-1B-Chat"}, "ip": "172.31.12.31"}
+{"tstamp": 1709203713.9642, "type": "chat", "model": "MobiLlama-1B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709203707.7742, "finish": 1709203713.9642, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "\u5e2e\u6211\u6574\u7406\u51fa\u6765"], ["Assistant", "\u6574\u7406\u51fa\u6765\u4e86\uff0c\u4ee5\u4e0b\u662f\u4e00\u4e9b\u5e38\u89c1\u7684\u6570\u636e\u7c7b\u578b\uff1a\n- \u6574\u6570\n- \u6d6e\u70b9\u6570\n- \u5b57\u7b26\u4e32\n- \u5e03\u5c14\u503c\n- \u5b57\u7b26\u4e32\u6570\u7ec4\n- \u6570\u7ec4\n- \u5bf9\u8c61\n- \u679a\u4e3e\n- \u6570\u7ec4\u548c\u5b57\u7b26\u4e32\u7684\u5bf9\u6bd4\n- \u6570\u7ec4\u548c\u5b57\u7b26\u4e32\u7684\u533a\u522b\n- \u5b57\u7b26\u4e32\u7684\u6bd4\u8f83\n- \u5b57\u7b26\u4e32\u7684\u622a\u65ad\n- \u5b57\u7b26\u4e32\u7684\u53bb\u9664\n- \u5b57\u7b26\u4e32\u7684\u8f6c\u4e49\n- \u5b57\u7b26\u4e32\u7684\u5339\u914d\n- \u5b57\u7b26\u4e32\u7684\u66ff\u6362\n- \u5b57\u7b26\u4e32\u7684\u5206\u6790\n- \u5b57\u7b26\u4e32\u7684\u8f6c\u5316\n- \u5b57\u7b26\u4e32\u7684\u6bd4\u8f83\n- \u5b57\u7b26\u4e32\u7684\u8f6c\u4e49\n- \u5b57\u7b26\u4e32\u7684\u5339\u914d\n- \u5b57\u7b26\u4e32\u7684\u66ff\u6362\n- \u5b57\u7b26\u4e32\u7684\u5206\u6790\n- \u5b57\u7b26\u4e32\u7684\u8f6c\u5316\n- \u5b57\u7b26\u4e32\u7684\u6bd4\u8f83\n- \u5b57\u7b26\u4e32\u7684\u8f6c\u4e49\n- \u5b57\u7b26\u4e32\u7684\u5339\u914d\n- \u5b57\u7b26\u4e32\u7684\u66ff\u6362\n- \u5b57\u7b26\u4e32\u7684\u5206\u6790\n- \u5b57\u7b26\u4e32\u7684\u8f6c\u5316\n- \u5b57\u7b26\u4e32\u7684\u6bd4\u8f83\n- \u5b57\u7b26\u4e32\u7684\u8f6c\u4e49\n- \u5b57\u7b26\u4e32\u7684\u5339\u914d\n- \u5b57\u7b26\u4e32\u7684\u66ff\u6362\n- \u5b57\u7b26\u4e32\u7684\u5206\u6790\n- \u5b57\u7b26\u4e32\u7684\u8f6c\u5316\n- \u5b57\u7b26\u4e32\u7684\u6bd4\u8f83\n- \u5b57\u7b26\u4e32\u7684\u8f6c\u4e49\n- \u5b57\u7b26\u4e32\u7684\ufffd"]], "offset": 0, "conv_id": "8450ba69ec064163b0997ff60744ca5e", "model_name": "MobiLlama-1B-Chat"}, "ip": "172.31.57.119"}
+{"tstamp": 1709203753.2842, "type": "chat", "model": "MobiLlama-1B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709203752.397, "finish": 1709203753.2842, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "\u5e2e\u6211\u6c47\u603b\u4e00\u4e0b\u8fd1\u534a\u5e74\u6765\u6b27\u5143\u5151\u4eba\u6c11\u5e01\u7684\u6c47\u7387"], ["Assistant", "\u6b27\u5143\u5151\u4eba\u6c11\u5e01\u7684\u6c47\u7387\u57282021\u5e746\u670826\u65e5\u81f32022\u5e741\u670825\u65e5\uff0c\u4e3a1:1\u3002"]], "offset": 0, "conv_id": "8450ba69ec064163b0997ff60744ca5e", "model_name": "MobiLlama-1B-Chat"}, "ip": "172.31.57.119"}
+{"tstamp": 1709214265.7959, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709214265.6141, "finish": 1709214265.7959, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "salut"], ["Assistant", "hello"]], "offset": 0, "conv_id": "783b077851054411b2e6fae2ed198e71", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.38.44"}
+{"tstamp": 1709237914.4236, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709237913.44, "finish": 1709237914.4236, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "who is barak obama"], ["Assistant", "Barack Obama is a U.S. politician and the 44th President of the United States. He was born on February 4, 1964, in Chicago, Illinois, USA. He is the 44th President of the United States."]], "offset": 0, "conv_id": "af723d77b3c642dc97a8312e2718dfe5", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.12.31"}
+{"tstamp": 1709237929.4318, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709237928.1712, "finish": 1709237929.4318, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "tell me more about him"], ["Assistant", "He is a 29-year-old man who has been in the music industry for over 10 years. He is known for his versatility in playing various instruments such as guitar, drums, and bass. He has a unique blend of traditional and modern music styles, and his unique sound has made him a popular musician in the industry."]], "offset": 0, "conv_id": "af723d77b3c642dc97a8312e2718dfe5", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.12.31"}
+{"tstamp": 1709237958.5662, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709237957.4541, "finish": 1709237958.5662, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "who are norse people"], ["Assistant", "Norse people are a group of people who lived in Norway, Sweden, Denmark, and Iceland in the 9th century. They were a group of warriors who fought for the power of the king and were known for their strength, skill, and courage. They were also known for their architecture and art."]], "offset": 0, "conv_id": "af723d77b3c642dc97a8312e2718dfe5", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.57.119"}
+{"tstamp": 1709237984.5959, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709237982.8487, "finish": 1709237984.5959, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "tell me more about their mythology"], ["Assistant", "As an AI language model, I don't have any personal mythology. However, I can tell you that mythology is a collection of stories and beliefs that have been passed down from generation to generation in different cultures. It is believed to have originated in ancient Greece and Rome and has evolved over time to include different stories and beliefs. Some of the most popular myths include the Hindu Mahabharata, the Greek myth of Poseidon, and the Norse myth of Odin."]], "offset": 0, "conv_id": "af723d77b3c642dc97a8312e2718dfe5", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.57.119"}
+{"tstamp": 1709238034.0094, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709238032.2459, "finish": 1709238034.0094, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "\"Bigger the better\" has been the predominant trend in recent Large Language Models (LLMs) development. However, LLMs do not suit well for scenarios that require on-device processing, energy efficiency, low memory footprint, and response efficiency. These requisites are crucial for privacy, security, and sustainable deployment. This paper explores the \"less is more\" paradigm by addressing the challenge of designing accurate yet efficient Small Language Models (SLMs) for resource constrained devices. Our primary contribution is the introduction of an accurate and fully transparent open-source 0.5 billion (0.5B) parameter SLM, named MobiLlama, catering to the specific needs of resource-constrained computing with an emphasis on enhanced performance with reduced resource demands. MobiLlama is a SLM design that initiates from a larger model and applies a careful parameter sharing scheme to reduce both the pre-training and the deployment cost. Our work strives to not only bridge the gap in open-source SLMs but also ensures full transparency, where complete training data pipeline, training code, model weights, and over 300 checkpoints along with evaluation codes is available at : https://github.com/mbzuai-oryx/MobiLlama.         based on the above context answer the question: how many parameters are present in the model"], ["Assistant", "MobiLlama is a 0.5 billion (0.5B) parameter SLM designed for resource-constrained computing. It is an open-source 0.5 billion (0.5B) parameter SLM that initiates from a larger model and applies a careful parameter sharing scheme to reduce both the pre-training and the deployment cost. It is designed to cater to the specific needs of resource-constrained computing with an emphasis on enhanced performance with reduced resource demands."]], "offset": 0, "conv_id": "af723d77b3c642dc97a8312e2718dfe5", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.12.31"}
+{"tstamp": 1709238128.8102, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709238128.3682, "finish": 1709238128.8102, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "Avagin Updates\nProgress:\n\u2022 Completed Roger Script: The core script, integrating voice recognition, text-\nto-speech, and a large language model (LLM), is now functional.\n\u2022 Flask Demo App: A user-friendly Flask-based application showcases the\nRoger script's capabilities.\n\u2022 Fine-Tuning and RAG Scripts Ready: Both fine-tuning scripts and Retrieval-\nAugmented Generation (RAG) scripts are prepared for domain-specific LLM\ndevelopment.\n\u2022 UI and Android App Demo Prepared: A basic Android app with user\ninterface is ready for integration with the LLM. Upon successful integration,\nwe can deliver a functional Android app demo within the next week for client\npresentation. This is not just a demo app; we plan to leverage this progress\nfor the final application.\nChallenges and Request:\n\u2022 GPU Requirement: Unfortunately, the fine-tuning and RAG processes require\nsignificant computational resources, specifically GPUs, which are currently\nunavailable.\n\u2022 Domain-Specific LLMs: We aim to create eight specialized LLMs, including\nthe Finance LLM designated for SatoshiAI. However, GPU access is crucial to\nachieve this objective.\nUrgent Request:\nTo ensure continued progress and meet development goals, we urgently request\naccess to one or more GPUs. This will enable us to:\n\u2022\n\u2022\n\u2022\nFine-tune LLMs: Tailor the LLMs to specific domains, including the Finance\nLLM for SatoshiAI.\nRun RAG Scripts: Enhance the LLMs' performance using the Retrieval-\nAugmented Generation technique.\nAchieve Domain-Specific Goals: Develop the eight planned LLMs, including\nthe critical Finance LLM for SatoshiAI.\nAndroid Integration:We understand the importance of Android integration and will continue to prioritize\nits development. Currently, we are investigating the most promising solutions to\novercome the challenges related to integrating Python libraries with Android,\nconsidering the following options:\nContributing to the Chocopy library: While considered a last resort due to\nits complexity, contributing to support the new Python libraries (llama-cpp-\npython and transformers) could be a potential solution.\n\u2022 TensorFlow Lite Approach: Although TensorFlow Lite supports some\nmodels, its current limitations include lack of phi2 support. Converting the\ngemma2B model to TFLite format might be viable, but requires access to a\npowerful GPU.\n\u2022 Kivy Library Exploration: While initial attempts with the Kivy library\nencountered errors, further investigation might yield solutions for building a\nmobile app.\n\u2022 Flutter Framework Evaluation: We are actively exploring the potential of\nFlutter's Langchain integration to enable LLM execution on mobile devices.\nFurther investigation into this option is warranted.\nAdditional Considerations:\n\u2022 Resource-Dependent App: We acknowledge that the Flask app is resource-\ndependent and requires a screen recording for demonstration purposes.\n\u2022 Final App Integration: We plan to leverage the current progress on the UI\nand Android app for the final application.\nNext Week's Tasks:\n\u2022 Continue investigating and working on Android integration: We will\nactively explore the mentioned solutions and prioritize the most promising\napproach based on feasibility and resource availability.\n\u2022 Integrate LLM with Android app: Upon receiving GPU access and identifying\na suitable solution, we will prioritize integrating the LLM with the Android app to\ndeliver the promised demo within the next week.\n\u2022 Integrate automation and API call for UBER ride booking: We will begin\nintegrating automation and API calls to facilitate ride booking with Uber\ndirectly through the Roger script.\n\u2022 Future Script Upgrades: Further enhancements are planned for the script,\nincluding streaming capabilities, bot memory, translation features, and\nimage/video input functionalities.\n\u2022Demo for Client:\n\u2022 Flask App Demo: While awaiting GPU access and Android integration, we\ncan showcase the Roger script's functionality through a screen recording of\nthe Flask app demo.\n\u2022 Android App Demo: Upon successful LLM integration, we will provide a\nfunctional Android app demo for client presentation within the next week.\nWe are confident that GPU access will significantly contribute to the success\nof the Roger project. We also remain committed to achieving Android\nintegration and delivering the promised demos as soon as possible.\n\nbased on the above context answer the question: what is the next weeks task?"], ["Assistant", "Next weeks task\nThe next weeks task is to continue investigating and working on Android integration."]], "offset": 0, "conv_id": "af723d77b3c642dc97a8312e2718dfe5", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.38.44"}
+{"tstamp": 1709245998.6928, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709245992.3665, "finish": 1709245998.6928, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "import argparse, os, sys, datetime, glob, importlib, csv\nimport numpy as np\nimport time\nimport torch\nimport torchvision\nimport pytorch_lightning as pl\n\nfrom packaging import version\nfrom omegaconf import OmegaConf\nfrom torch.utils.data import random_split, DataLoader, Dataset, Subset\nfrom functools import partial\nfrom PIL import Image\n\nfrom pytorch_lightning import seed_everything\nfrom pytorch_lightning.trainer import Trainer\nfrom pytorch_lightning.callbacks import ModelCheckpoint, Callback, LearningRateMonitor\nfrom pytorch_lightning.utilities.distributed import rank_zero_only\nfrom pytorch_lightning.utilities import rank_zero_info\n\nfrom ldm.data.base import Txt2ImgIterableBaseDataset\nfrom ldm.util import instantiate_from_config\n\n\ndef get_parser(**parser_kwargs):\n    def str2bool(v):\n        if isinstance(v, bool):\n            return v\n        if v.lower() in (\"yes\", \"true\", \"t\", \"y\", \"1\"):\n            return True\n        elif v.lower() in (\"no\", \"false\", \"f\", \"n\", \"0\"):\n            return False\n        else:\n            raise argparse.ArgumentTypeError(\"Boolean value expected.\")\n\n    parser = argparse.ArgumentParser(**parser_kwargs)\n    parser.add_argument(\n        \"-n\",\n        \"--name\",\n        type=str,\n        const=True,\n        default=\"\",\n        nargs=\"?\",\n        help=\"postfix for logdir\",\n    )\n    parser.add_argument(\n        \"-r\",\n        \"--resume\",\n        type=str,\n        const=True,\n        default=\"\",\n        nargs=\"?\",\n        help=\"resume from logdir or checkpoint in logdir\",\n    )\n    parser.add_argument(\n        \"-b\",\n        \"--base\",\n        nargs=\"*\",\n        metavar=\"base_config.yaml\",\n        help=\"paths to base configs. Loaded from left-to-right. \"\n             \"Parameters can be overwritten or added with command-line options of the form `--key value`.\",\n        default=list(),\n    )\n    parser.add_argument(\n        \"-t\",\n        \"--train\",\n        type=str2bool,\n        const=True,\n        default=False,\n        nargs=\"?\",\n        help=\"train\",\n    )\n    parser.add_argument(\n        \"--no-test\",\n        type=str2bool,\n        const=True,\n        default=False,\n        nargs=\"?\",\n        help=\"disable test\",\n    )\n    parser.add_argument(\n        \"-p\",\n        \"--project\",\n        help=\"name of new or path to existing project\"\n    )\n    parser.add_argument(\n        \"-d\",\n        \"--debug\",\n        type=str2bool,\n        nargs=\"?\",\n        const=True,\n        default=False,\n        help=\"enable post-mortem debugging\",\n    )\n    parser.add_argument(\n        \"-s\",\n        \"--seed\",\n        type=int,\n        default=23,\n        help=\"seed for seed_everything\",\n    )\n    parser.add_argument(\n        \"-f\",\n        \"--postfix\",\n        type=str,\n        default=\"\",\n        help=\"post-postfix for default name\",\n    )\n    parser.add_argument(\n        \"-l\",\n        \"--logdir\",\n        type=str,\n        default=\"logs\",\n        help=\"directory for logging dat shit\",\n    )\n    parser.add_argument(\n        \"--scale_lr\",\n        type=str2bool,\n        nargs=\"?\",\n        const=True,\n        default=True,\n        help=\"scale base-lr by ngpu * batch_size * n_accumulate\",\n    )\n    return parser\n\n\ndef nondefault_trainer_args(opt):\n    parser = argparse.ArgumentParser()\n    parser = Trainer.add_argparse_args(parser)\n    args = parser.parse_args([])\n    return sorted(k for k in vars(args) if getattr(opt, k) != getattr(args, k))\n\n\nclass WrappedDataset(Dataset):\n    \"\"\"Wraps an arbitrary object with __len__ and __getitem__ into a pytorch dataset\"\"\"\n\n    def __init__(self, dataset):\n        self.data = dataset\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        return self.data[idx]\n\n\ndef worker_init_fn(_):\n    worker_info = torch.utils.data.get_worker_info()\n\n    dataset = worker_info.dataset\n    worker_id = worker_info.id\n\n    if isinstance(dataset, Txt2ImgIterableBaseDataset):\n        split_size = dataset.num_records // worker_info.num_workers\n        # reset num_records to the true number to retain reliable length information\n        dataset.sample_ids = dataset.valid_ids[worker_id * split_size:(worker_id + 1) * split_size]\n        current_id = np.random.choice(len(np.random.get_state()[1]), 1)\n        return np.random.seed(np.random.get_state()[1][current_id] + worker_id)\n    else:\n        return np.random.seed(np.random.get_state()[1][0] + worker_id)\n\n\nclass DataModuleFromConfig(pl.LightningDataModule):\n    def __init__(self, batch_size, train=None, validation=None, test=None, predict=None,\n                 wrap=False, num_workers=None, shuffle_test_loader=False, use_worker_init_fn=False,\n                 shuffle_val_dataloader=False):\n        super().__init__()\n        self.batch_size = batch_size\n        self.dataset_configs = dict()\n        self.num_workers = num_workers if num_workers is not None else batch_size * 2\n        self.use_worker_init_fn = use_worker_init_fn\n        if train is not None:\n            self.dataset_configs[\"train\"] = train\n            self.train_dataloader = self._train_dataloader\n        if validation is not None:\n            self.dataset_configs[\"validation\"] = validation\n            self.val_dataloader = partial(self._val_dataloader, shuffle=shuffle_val_dataloader)\n        if test is not None:\n            self.dataset_configs[\"test\"] = test\n            self.test_dataloader = partial(self._test_dataloader, shuffle=shuffle_test_loader)\n        if predict is not None:\n            self.dataset_configs[\"predict\"] = predict\n            self.predict_dataloader = self._predict_dataloader\n        self.wrap = wrap\n\n    def prepare_data(self):\n        for data_cfg in self.dataset_configs.values():\n            instantiate_from_config(data_cfg)\n\n    def setup(self, stage=None):\n        self.datasets = dict(\n            (k, instantiate_from_config(self.dataset_configs[k]))\n            for k in self.dataset_configs)\n        if self.wrap:\n            for k in self.datasets:\n                self.datasets[k] = WrappedDataset(self.datasets[k])\n\n    def _train_dataloader(self):\n        is_iterable_dataset = isinstance(self.datasets['train'], Txt2ImgIterableBaseDataset)\n        if is_iterable_dataset or self.use_worker_init_fn:\n            init_fn = worker_init_fn\n        else:\n            init_fn = None\n        return DataLoader(self.datasets[\"train\"], batch_size=self.batch_size,\n                          num_workers=self.num_workers, shuffle=False if is_iterable_dataset else True,\n                          worker_init_fn=init_fn)\n\n    def _val_dataloader(self, shuffle=False):\n        if isinstance(self.datasets['validation'], Txt2ImgIterableBaseDataset) or self.use_worker_init_fn:\n            init_fn = worker_init_fn\n        else:\n            init_fn = None\n        return DataLoader(self.datasets[\"validation\"],\n                          batch_size=self.batch_size,\n                          num_workers=self.num_workers,\n                          worker_init_fn=init_fn,\n                          shuffle=shuffle)\n\n    def _test_dataloader(self, shuffle=False):\n        is_iterable_dataset = isinstance(self.datasets['train'], Txt2ImgIterableBaseDataset)\n        if is_iterable_dataset or self.use_worker_init_fn:\n            init_fn = worker_init_fn\n        else:\n            init_fn = None\n\n        # do not shuffle dataloader for iterable dataset\n        shuffle = shuffle and (not is_iterable_dataset)\n\n        return DataLoader(self.datasets[\"test\"], batch_size=self.batch_size,\n                          num_workers=self.num_workers, worker_init_fn=init_fn, shuffle=shuffle)\n\n    def _predict_dataloader(self, shuffle=False):\n        if isinstance(self.datasets['predict'], Txt2ImgIterableBaseDataset) or self.use_worker_init_fn:\n            init_fn = worker_init_fn\n        else:\n            init_fn = None\n        return DataLoader(self.datasets[\"predict\"], batch_size=self.batch_size,\n                          num_workers=self.num_workers, worker_init_fn=init_fn)\n\n\nclass SetupCallback(Callback):\n    def __init__(self, resume, now, logdir, ckptdir, cfgdir, config, lightning_config):\n        super().__init__()\n        self.resume = resume\n        self.now = now\n        self.logdir = logdir\n        self.ckptdir = ckptdir\n        self.cfgdir = cfgdir\n        self.config = config\n        self.lightning_config = lightning_config\n\n    def on_keyboard_interrupt(self, trainer, pl_module):\n        if trainer.global_rank == 0:\n            print(\"Summoning checkpoint.\")\n            ckpt_path = os.path.join(self.ckptdir, \"last.ckpt\")\n            trainer.save_checkpoint(ckpt_path)\n\n    def on_pretrain_routine_start(self, trainer, pl_module):\n        if trainer.global_rank == 0:\n            # Create logdirs and save configs\n            os.makedirs(self.logdir, exist_ok=True)\n            os.makedirs(self.ckptdir, exist_ok=True)\n            os.makedirs(self.cfgdir, exist_ok=True)\n\n            if \"callbacks\" in self.lightning_config:\n                if 'metrics_over_trainsteps_checkpoint' in self.lightning_config['callbacks']:\n                    os.makedirs(os.path.join(self.ckptdir, 'trainstep_checkpoints'), exist_ok=True)\n            print(\"Project config\")\n            print(OmegaConf.to_yaml(self.config))\n            OmegaConf.save(self.config,\n                           os.path.join(self.cfgdir, \"{}-project.yaml\".format(self.now)))\n\n            print(\"Lightning config\")\n            print(OmegaConf.to_yaml(self.lightning_config))\n            OmegaConf.save(OmegaConf.create({\"lightning\": self.lightning_config}),\n                           os.path.join(self.cfgdir, \"{}-lightning.yaml\".format(self.now)))\n\n        else:\n            # ModelCheckpoint callback created log directory --- remove it\n            if not self.resume and os.path.exists(self.logdir):\n                dst, name = os.path.split(self.logdir)\n                dst = os.path.join(dst, \"child_runs\", name)\n                os.makedirs(os.path.split(dst)[0], exist_ok=True)\n                try:\n                    os.rename(self.logdir, dst)\n                except FileNotFoundError:\n                    pass\n\n\nclass ImageLogger(Callback):\n    def __init__(self, batch_frequency, max_images, clamp=True, increase_log_steps=True,\n                 rescale=True, disabled=False, log_on_batch_idx=False, log_first_step=False,\n                 log_images_kwargs=None):\n        super().__init__()\n        self.rescale = rescale\n        self.batch_freq = batch_frequency\n        self.max_images = max_images\n        self.logger_log_images = {\n            pl.loggers.TestTubeLogger: self._testtube,\n        }\n        self.log_steps = [2 ** n for n in range(int(np.log2(self.batch_freq)) + 1)]\n        if not increase_log_steps:\n            self.log_steps = [self.batch_freq]\n        self.clamp = clamp\n        self.disabled = disabled\n        self.log_on_batch_idx = log_on_batch_idx\n        self.log_images_kwargs = log_images_kwargs if log_images_kwargs else {}\n        self.log_first_step = log_first_step\n\n    @rank_zero_only\n    def _testtube(self, pl_module, images, batch_idx, split):\n        for k in images:\n            grid = torchvision.utils.make_grid(images[k])\n            grid = (grid + 1.0) / 2.0  # -1,1 -> 0,1; c,h,w\n\n            tag = f\"{split}/{k}\"\n            pl_module.logger.experiment.add_image(\n                tag, grid,\n                global_step=pl_module.global_step)\n\n    @rank_zero_only\n    def log_local(self, save_dir, split, images,\n                  global_step, current_epoch, batch_idx):\n        root = os.path.join(save_dir, \"images\", split)\n        for k in images:\n            grid = torchvision.utils.make_grid(images[k], nrow=4)\n            if self.rescale:\n                grid = (grid + 1.0) / 2.0  # -1,1 -> 0,1; c,h,w\n            grid = grid.transpose(0, 1).transpose(1, 2).squeeze(-1)\n            grid = grid.numpy()\n            grid = (grid * 255).astype(np.uint8)\n            filename = \"{}_gs-{:06}_e-{:06}_b-{:06}.png\".format(\n                k,\n                global_s"], ["Assistant", "ImageLogger class\n        pl_module = ImageLogger(batch_frequency=self.batch_freq, max_images=self.max_images,\n                                 clamp=self.clamp, increase_log_steps=self.increase_log_steps,\n                                 rescale=self.rescale, disabled=self.disabled,\n                                 log_on_batch_idx=self.log_on_batch_idx,\n                                 log_first_step=self.log_first_step)\n        pl_module.logger.experiment.add_image(\"\", root, global_step=current_epoch, batch_idx=batch_idx)\n        return pl_module\n\n    def _checkpoint(self, pl_module, save_dir, current_epoch, batch_idx,\n                      max_images,\n                      disable_training,\n                      log_on_batch_idx,\n                      log_first_step,\n                      log_images_kwargs):\n        if not self.rescale and self.max_images > 1:\n            # Rescale images\n            grid = torchvision.utils.make_grid(self.max_images, nrow=4)\n            grid = grid.transpose(0, 1).transpose(1, 2).squeeze(-1)\n            grid = grid.numpy()\n            grid = (grid * 255).astype(np.uint8)\n            filename = \"{}_gs-{:06}_e-{:06}_b-{:06}.png\".format(\n                self.max_images,\n                self.batch_freq,\n                self.max_images,\n                self.batch_freq,"]], "offset": 0, "conv_id": "ce306a0a36f14207b7508fd11c60e878", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.12.31"}
+{"tstamp": 1709246024.7416, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709246018.4757, "finish": 1709246024.7416, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "find errors in the above code"], ["Assistant", "There are several errors in the code that need to be fixed. Here are a few:\n1. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n2. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n3. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n4. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n5. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n6. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n7. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n8. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n9. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n10. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n11. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n12. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n13. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n14. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n15. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n16. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n17. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n18. `print(\"Hello, World!\")` should be `print(\""]], "offset": 0, "conv_id": "ce306a0a36f14207b7508fd11c60e878", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.12.31"}
+{"tstamp": 1709246105.9188, "type": "downvote", "model": "MobiLlama-05B-Chat", "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "find errors in the above code"], ["Assistant", "There are several errors in the code that need to be fixed. Here are a few:\n1. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n2. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n3. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n4. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n5. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n6. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n7. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n8. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n9. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n10. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n11. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n12. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n13. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n14. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n15. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n16. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n17. `print(\"Hello, World!\")` should be `print(\"Hello, World!\")`\n18. `print(\"Hello, World!\")` should be `print(\""]], "offset": 0, "conv_id": "ce306a0a36f14207b7508fd11c60e878", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.57.119"}
+{"tstamp": 1709246120.4847, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709246117.8915, "finish": 1709246120.4847, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "find errors in the above code"], ["Assistant", "As an AI language model, I don't have access to the code you are referring to. However, I can provide you with some general guidelines for checking for errors in code.\n1. Use a code editor or IDE to check for errors.\n2. Use a debugger to step through the code and identify any errors.\n3. Use a code linter to check for syntax errors, such as missing parentheses, missing semicolons, or incorrect variable names.\n4. Use a code quality checker to identify any coding style issues, such as unused variables or excessive indentation.\n5. Use a code review tool to identify any potential issues with the code.\n6. Use a code review team to review the code and provide feedback on any issues identified."]], "offset": 0, "conv_id": "ce306a0a36f14207b7508fd11c60e878", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.57.119"}

2024-03-01-conv.json

@@ -0,0 +1,8 @@
+{"tstamp": 1709252309.8682, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709252307.0823, "finish": 1709252309.8682, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "Who is Prof Yi-Zhe Song? What is his research background? "], ["Assistant", "Prof Yi-Zhe Song is a Professor of Electrical Engineering and Computer Science at the University of Texas at Austin. He received his Ph.D. in Electrical Engineering from the University of California, Berkeley, in 2004. He is also a Senior Research Scientist at the University of Texas at Austin. Prof Song's research interests include signal processing, machine learning, and optimization. He has published over 300 papers in top-tier conferences and journals, including IEEE Transactions on Pattern Analysis and Machine Intelligence, IEEE Transactions on Signal Processing, and IEEE Transactions on Knowledge and Data Engineering. Prof Song is a recipient of numerous awards and grants, including the National Science Foundation Graduate Research Fellowship and the National Institutes of Health Young Investigator Award."]], "offset": 0, "conv_id": "d5bad730cd4b48bab48a80156b991ebd", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.57.119"}
+{"tstamp": 1709252330.9337, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709252330.4529, "finish": 1709252330.9337, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "can you give any link?"], ["Assistant", "Sure, here is the link: https://www.google.com/search?q=google+search+api"]], "offset": 0, "conv_id": "d5bad730cd4b48bab48a80156b991ebd", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.57.119"}
+{"tstamp": 1709252352.6014, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709252351.7288, "finish": 1709252352.6014, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "do you know Ayan Kumar Bhunia"], ["Assistant", "Yes, Ayan Kumar Bhunia is a famous Indian film actor, producer, and director. He is known for his role in the Bollywood film 'Dil Dosti Duniya Do' and 'Meri Aashiqui'."]], "offset": 0, "conv_id": "d5bad730cd4b48bab48a80156b991ebd", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.57.119"}
+{"tstamp": 1709261178.0713, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709261171.7866, "finish": 1709261178.0713, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "\u5c06\u4e0b\u9762\u6587\u4ef6\u7ffb\u8bd1\u6210\u4e2d\u6587 \u201cDescribe the bug\nI'm seeing 5x i/s improvements in Ooba when using the exLlama loading and was hopeful that it would help serve up some gains when using ooba's multimodal extension.\n\nWhen attempting to use exllama as the loader with any of the 4 stock Multimodal setups in either chat, instruct-chat or instruct modesl, text will work on its own but adding an image to the multimodal extension results in this error.\n\nMaybe you can point me in the right direction to try and resolve this?\n\nIs there an existing issue for this?\n[X] I have searched the existing issues\nReproduction\nDownload Multimodal models Install miniGPT if necessary Install exLlama repository (now a part of the latest one click isntall)\n\nScreenshot\nNo response\n\nLogs\nFile \"D:\\00\\text-generation-webui\\modules\\text_generation.py\", line 300, in generate_reply_custom\n    for reply in shared.model.generate_with_streaming(question, state):\n  File \"D:\\00\\text-generation-webui\\modules\\exllama.py\", line 68, in generate_with_streaming\n    self.generator.gen_begin_reuse(ids)\n  File \"D:\\00\\text-generation-webui\\repositories\\exllama\\generator.py\", line 191, in gen_begin_reuse\n    if reuse < in_tokens.shape[-1]: self.gen_feed_tokens(in_tokens[:, reuse:])\n  File \"D:\\00\\text-generation-webui\\repositories\\exllama\\generator.py\", line 209, in gen_feed_tokens\n    self.model.forward(self.sequence[:, start:-1], self.cache, preprocess_only = True, lora = self.lora)\n  File \"D:\\00\\text-generation-webui\\repositories\\exllama\\model.py\", line 841, in forward\n    hidden_states = decoder_layer.forward(hidden_states, cache, buffers[device], lora)\n  File \"D:\\00\\text-generation-webui\\repositories\\exllama\\model.py\", line 459, in forward\n    hidden_states = self.self_attn.forward(hidden_states, cache, buffer, lora)\n  File \"D:\\00\\text-generation-webui\\repositories\\exllama\\model.py\", line 381, in forward\n    new_keys = cache.key_states[self.index].narrow(2, past_len, q_len)\nRuntimeError: start (49) + length (13970) exceeds dimension size (2048).\nOutput generated in 1.51 seconds (0.00 tokens/s, 0 tokens, context 14020, seed 979644525)\nSystem Info\nWindows 10, 96 GB RAM, Nvidia GTX 2080i 11GB\nAlso confirmed same issue on\nWindows 11, Titan RTX 24 GB\u201d"], ["Assistant", "\u5728\u4e0b\u9762\u6587\u4ef6\u4e2d\u7ffb\u8bd1\u6210\u4e2d\u6587 \u201cDescribe the bug\u201d\n\n\u6587\u4ef6\uff1a\u201cDescribe the bug\u201d\n\nIssue\uff1a\n\n1. \u4f7f\u7528 ooba \u7684 multimodal \u6a21\u578b\u65f6\uff0c\u5982\u679c\u5728\u4f7f\u7528 ooba \u7684 multimodal \u6a21\u578b\u65f6\uff0c\u4f7f\u7528 ooba \u7684 exLlama \u6a21\u578b\uff0c\u5c06\u4f1a\u51fa\u73b0\u4ee5\u4e0b\u9519\u8bef\uff1a\n\n```\nFile \"D:\\00\\text-generation-webui\\modules\\text_generation.py\", line 300, in generate_reply_custom\n    for reply in shared.model.generate_with_streaming(question, state):\n  File \"D:\\00\\text-generation-webui\\modules\\exllama.py\", line 68, in generate_with_streaming\n    self.generator.gen_begin_reuse(ids)\n  File \"D:\\00\\text-generation-webui\\repositories\\exllama\\generator.py\", line 191, in gen_begin_reuse\n    if reuse < in_tokens.shape[-1]: self.gen_feed_tokens(in_tokens[:, reuse:])\n  File \"D:\\00\\text-generation-webui\\repositories\\exllama\\generator.py\", line 209, in gen_feed_tokens\n    if reuse < in_tokens.shape[-1]: self.gen_feed_tokens(in_tokens[:, reuse:])\n  File \"D:\\00\\text-generation-webui\\repositories\\exllama\\generator.py\", line 209, in gen"]], "offset": 0, "conv_id": "bff45589528e4269a2e8ae0d31c801bd", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.18.237"}
+{"tstamp": 1709261335.2558, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709261332.0782, "finish": 1709261335.2558, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "\u7ffb\u8bd1\u6210\u4e2d\u6587\u201cDescribe the bug\nI'm seeing 5x i/s improvements in Ooba when using the exLlama loading and was hopeful that it would help serve up some gains when using ooba's multimodal extension.\n\nWhen attempting to use exllama as the loader with any of the 4 stock Multimodal setups in either chat, instruct-chat or instruct modesl, text will work on its own but adding an image to the multimodal extension results in this error.\n\nMaybe you can point me in the right direction to try and resolve this?\n\nIs there an existing issue for this?\n[X] I have searched the existing issues\nReproduction\nDownload Multimodal models Install miniGPT if necessary Install exLlama repository (now a part of the latest one click isntall)\n\nScreenshot\nNo response\u201d"], ["Assistant", "\u7ffb\u8bd1\u6210\u4e2d\u6587\u201cI'm seeing 5x i/s improvements in Ooba when using the exLlama loading and was hopeful that it would help serve up some gains when using ooba's multimodal extension.\n\nWhen attempting to use exllama as the loader with any of the 4 stock Multimodal setups in either chat, instruct-chat or instruct modesl, text will work on its own but adding an image to the multimodal extension results in this error.\n\nMaybe you can point me in the right direction to try and resolve this?\n\nIs there an existing issue for this?\n[X] I have searched the existing issues\nReproduction\nDownload Multimodal models Install miniGPT if necessary Install exLlama repository (now a part of the latest one click isntall)\n\nScreenshot\nNo response\u201d"]], "offset": 0, "conv_id": "bff45589528e4269a2e8ae0d31c801bd", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.18.237"}
+{"tstamp": 1709261369.0236, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709261362.7107, "finish": 1709261369.0236, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "\u5c06\u4e0b\u9762\u53cc\u5f15\u53f7\u4e2d\u7684\u6587\u5b57\u7ffb\u8bd1\u6210\u4e2d\u6587\u201cDescribe the bug\nI'm seeing 5x i/s improvements in Ooba when using the exLlama loading and was hopeful that it would help serve up some gains when using ooba's multimodal extension.\n\nWhen attempting to use exllama as the loader with any of the 4 stock Multimodal setups in either chat, instruct-chat or instruct modesl, text will work on its own but adding an image to the multimodal extension results in this error.\n\nMaybe you can point me in the right direction to try and resolve this?\n\nIs there an existing issue for this?\n[X] I have searched the existing issues\nReproduction\nDownload Multimodal models Install miniGPT if necessary Install exLlama repository (now a part of the latest one click isntall)\n\nScreenshot\nNo response\u201d"], ["Assistant", "\u6211\u53ef\u4ee5\u4e3a\u4f60\u63d0\u4f9b\u8be6\u7ec6\u7684\u89e3\u51b3\u65b9\u6848\u3002\n\u5982\u679c\u4f60\u662f\u5426\u6709\u4efb\u4f55\u95ee\u9898\uff0c\u8bf7\u95ee\u4f60\u6709\u600e\u4e48\u89e3\u51b3\u8fd9\u4e2a\u95ee\u9898\uff1f\n[X] I have searched the existing issues\n[ ] I have found an existing issue\n[ ] I have searched for a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found a solution to this problem\n[ ] I have found"]], "offset": 0, "conv_id": "bff45589528e4269a2e8ae0d31c801bd", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.18.237"}
+{"tstamp": 1709261390.198, "type": "chat", "model": "MobiLlama-05B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709261389.3716, "finish": 1709261390.198, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "\u5c06\u4e0b\u9762\u53cc\u5f15\u53f7\u4e2d\u7684\u6587\u5b57\u7ffb\u8bd1\u6210\u4e2d\u6587\u201cDescribe the bug\nI'm seeing 5x i/s improvements in Ooba when using the exLlama loading and was hopeful that it would help serve up some gains when using ooba's multimodal extension.\n\nWhen attempting to use exllama as the loader with any of the 4 stock Multimodal setups in either chat, instruct-chat or instruct modesl, text will work on its own but adding an image to the multimodal extension results in this error.\n\nMaybe you can point me in the right direction to try and resolve this?\n\nIs there an existing issue for this?\n[X] I have searched the existing issues\nReproduction\nDownload Multimodal models Install miniGPT if necessary Install exLlama repository (now a part of the latest one click isntall)\n\nScreenshot\nNo response\u201d"], ["Assistant", "I'm sorry, but I cannot provide a response as I do not have access to your account or any other relevant information. Can you please provide me with the information you mentioned so that I can assist you further?"]], "offset": 0, "conv_id": "bff45589528e4269a2e8ae0d31c801bd", "model_name": "MobiLlama-05B-Chat"}, "ip": "172.31.18.237"}
+{"tstamp": 1709279504.3264, "type": "chat", "model": "MobiLlama-1B-Chat", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1709279504.0812, "finish": 1709279504.3264, "state": {"template_name": "zero_shot", "system_message": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", "roles": ["Human", "Assistant"], "messages": [["Human", "hello"], ["Assistant", "Hello! How can I assist you today?"]], "offset": 0, "conv_id": "c8a58b9ebd9e42e7b5b2e01cd648829d", "model_name": "MobiLlama-1B-Chat"}, "ip": "172.31.38.44"}

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,12 +1,358 @@
 ---
 title: MobiLlama
-emoji: 🔥
-colorFrom: purple
-colorTo: green
+app_file: fastchat/serve/gradio_web_server.py
 sdk: gradio
-sdk_version: 4.19.2
-app_file: app.py
-pinned: false
+sdk_version: 3.50.2
 ---
+# FastChat
+| [**Demo**](https://chat.lmsys.org/) | [**Discord**](https://discord.gg/HSWAKCrnFx) | [**X**](https://x.com/lmsysorg) |
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+FastChat is an open platform for training, serving, and evaluating large language model based chatbots.
+- FastChat powers Chatbot Arena (https://chat.lmsys.org/), serving over 6 million chat requests for 50+ LLMs.
+- Chatbot Arena has collected over 200K human votes from side-by-side LLM battles to compile an online [LLM Elo leaderboard](https://huggingface.co/spaces/lmsys/chatbot-arena-leaderboard).
+
+FastChat's core features include:
+- The training and evaluation code for state-of-the-art models (e.g., Vicuna, MT-Bench).
+- A distributed multi-model serving system with web UI and OpenAI-compatible RESTful APIs.
+
+## News
+- [2023/09] 🔥 We released **LMSYS-Chat-1M**, a large-scale real-world LLM conversation dataset. Read the [report](https://arxiv.org/abs/2309.11998).
+- [2023/08] We released **Vicuna v1.5** based on Llama 2 with 4K and 16K context lengths. Download [weights](#vicuna-weights).
+- [2023/07] We released **Chatbot Arena Conversations**, a dataset containing 33k conversations with human preferences. Download it [here](https://huggingface.co/datasets/lmsys/chatbot_arena_conversations).
+
+<details>
+<summary>More</summary>
+
+- [2023/08] We released **LongChat v1.5** based on Llama 2 with 32K context lengths. Download [weights](#longchat).
+- [2023/06] We introduced **MT-bench**, a challenging multi-turn question set for evaluating chatbots. Check out the blog [post](https://lmsys.org/blog/2023-06-22-leaderboard/).
+- [2023/06] We introduced **LongChat**, our long-context chatbots and evaluation tools. Check out the blog [post](https://lmsys.org/blog/2023-06-29-longchat/).
+- [2023/05] We introduced **Chatbot Arena** for battles among LLMs. Check out the blog [post](https://lmsys.org/blog/2023-05-03-arena).
+- [2023/03] We released **Vicuna: An Open-Source Chatbot Impressing GPT-4 with 90% ChatGPT Quality**. Check out the blog [post](https://vicuna.lmsys.org).
+
+</details>
+
+<a href="https://chat.lmsys.org"><img src="assets/demo_narrow.gif" width="70%"></a>
+
+## Contents
+- [Install](#install)
+- [Model Weights](#model-weights)
+- [Inference with Command Line Interface](#inference-with-command-line-interface)
+- [Serving with Web GUI](#serving-with-web-gui)
+- [API](#api)
+- [Evaluation](#evaluation)
+- [Fine-tuning](#fine-tuning)
+- [Citation](#citation)
+
+## Install
+
+### Method 1: With pip
+
+```bash
+pip3 install "fschat[model_worker,webui]"
+```
+
+### Method 2: From source
+
+1. Clone this repository and navigate to the FastChat folder
+```bash
+git clone https://github.com/lm-sys/FastChat.git
+cd FastChat
+```
+
+If you are running on Mac:
+```bash
+brew install rust cmake
+```
+
+2. Install Package
+```bash
+pip3 install --upgrade pip  # enable PEP 660 support
+pip3 install -e ".[model_worker,webui]"
+```
+
+## Model Weights
+### Vicuna Weights
+[Vicuna](https://lmsys.org/blog/2023-03-30-vicuna/) is based on Llama 2 and should be used under Llama's [model license](https://github.com/facebookresearch/llama/blob/main/LICENSE).
+
+You can use the commands below to start chatting. It will automatically download the weights from Hugging Face repos.
+Downloaded weights are stored in a `.cache` folder in the user's home folder (e.g., `~/.cache/huggingface/hub/<model_name>`).
+
+See more command options and how to handle out-of-memory in the "Inference with Command Line Interface" section below.
+
+**NOTE: `transformers>=4.31` is required for 16K versions.**
+
+| Size | Chat Command | Hugging Face Repo |
+| ---  | --- | --- |
+| 7B   | `python3 -m fastchat.serve.cli --model-path lmsys/vicuna-7b-v1.5`  | [lmsys/vicuna-7b-v1.5](https://huggingface.co/lmsys/vicuna-7b-v1.5)   |
+| 7B-16k   | `python3 -m fastchat.serve.cli --model-path lmsys/vicuna-7b-v1.5-16k`  | [lmsys/vicuna-7b-v1.5-16k](https://huggingface.co/lmsys/vicuna-7b-v1.5-16k)   |
+| 13B  | `python3 -m fastchat.serve.cli --model-path lmsys/vicuna-13b-v1.5` | [lmsys/vicuna-13b-v1.5](https://huggingface.co/lmsys/vicuna-13b-v1.5) |
+| 13B-16k  | `python3 -m fastchat.serve.cli --model-path lmsys/vicuna-13b-v1.5-16k` | [lmsys/vicuna-13b-v1.5-16k](https://huggingface.co/lmsys/vicuna-13b-v1.5-16k) |
+| 33B  | `python3 -m fastchat.serve.cli --model-path lmsys/vicuna-33b-v1.3` | [lmsys/vicuna-33b-v1.3](https://huggingface.co/lmsys/vicuna-33b-v1.3) |
+
+**Old weights**: see [docs/vicuna_weights_version.md](docs/vicuna_weights_version.md) for all versions of weights and their differences.
+
+### Other Models
+Besides Vicuna, we also released two additional models: [LongChat](https://lmsys.org/blog/2023-06-29-longchat/) and FastChat-T5.
+You can use the commands below to chat with them. They will automatically download the weights from Hugging Face repos.
+
+| Model | Chat Command | Hugging Face Repo |
+| ---  | --- | --- |
+| LongChat-7B   | `python3 -m fastchat.serve.cli --model-path lmsys/longchat-7b-32k-v1.5`  | [lmsys/longchat-7b-32k](https://huggingface.co/lmsys/longchat-7b-32k-v1.5)   |
+| FastChat-T5-3B   | `python3 -m fastchat.serve.cli --model-path lmsys/fastchat-t5-3b-v1.0`  | [lmsys/fastchat-t5-3b-v1.0](https://huggingface.co/lmsys/fastchat-t5-3b-v1.0) |
+
+## Inference with Command Line Interface
+
+<a href="https://chat.lmsys.org"><img src="assets/screenshot_cli.png" width="70%"></a>
+
+(Experimental Feature: You can specify `--style rich` to enable rich text output and better text streaming quality for some non-ASCII content. This may not work properly on certain terminals.)
+
+#### Supported Models
+FastChat supports a wide range of models, including
+LLama 2, Vicuna, Alpaca, Baize, ChatGLM, Dolly, Falcon, FastChat-T5, GPT4ALL, Guanaco, MTP, OpenAssistant, OpenChat, RedPajama, StableLM, WizardLM, xDAN-AI and more.
+
+See a complete list of supported models and instructions to add a new model [here](docs/model_support.md).
+
+#### Single GPU
+The command below requires around 14GB of GPU memory for Vicuna-7B and 28GB of GPU memory for Vicuna-13B.
+See the ["Not Enough Memory" section](#not-enough-memory) below if you do not have enough memory.
+`--model-path` can be a local folder or a Hugging Face repo name.
+```
+python3 -m fastchat.serve.cli --model-path lmsys/vicuna-7b-v1.5
+```
+
+#### Multiple GPUs
+You can use model parallelism to aggregate GPU memory from multiple GPUs on the same machine. 
+```
+python3 -m fastchat.serve.cli --model-path lmsys/vicuna-7b-v1.5 --num-gpus 2
+```
+
+Tips:
+Sometimes the "auto" device mapping strategy in huggingface/transformers does not perfectly balance the memory allocation across multiple GPUs.
+You can use `--max-gpu-memory` to specify the maximum memory per GPU for storing model weights.
+This allows it to allocate more memory for activations, so you can use longer context lengths or larger batch sizes. For example,
+
+```
+python3 -m fastchat.serve.cli --model-path lmsys/vicuna-7b-v1.5 --num-gpus 2 --max-gpu-memory 8GiB
+```
+
+#### CPU Only
+This runs on the CPU only and does not require GPU. It requires around 30GB of CPU memory for Vicuna-7B and around 60GB of CPU memory for Vicuna-13B.
+```
+python3 -m fastchat.serve.cli --model-path lmsys/vicuna-7b-v1.5 --device cpu
+```
+
+Use Intel AI Accelerator AVX512_BF16/AMX to accelerate CPU inference.
+```
+CPU_ISA=amx python3 -m fastchat.serve.cli --model-path lmsys/vicuna-7b-v1.5 --device cpu
+```
+
+#### Metal Backend (Mac Computers with Apple Silicon or AMD GPUs)
+Use `--device mps` to enable GPU acceleration on Mac computers (requires torch >= 2.0).
+Use `--load-8bit` to turn on 8-bit compression.
+```
+python3 -m fastchat.serve.cli --model-path lmsys/vicuna-7b-v1.5 --device mps --load-8bit
+```
+Vicuna-7B can run on a 32GB M1 Macbook with 1 - 2 words / second.
+
+#### Intel XPU (Intel Data Center and Arc A-Series GPUs)
+Install the [Intel Extension for PyTorch](https://intel.github.io/intel-extension-for-pytorch/xpu/latest/tutorials/installation.html). Set the OneAPI environment variables:
+```
+source /opt/intel/oneapi/setvars.sh
+```
+
+Use `--device xpu` to enable XPU/GPU acceleration.
+```
+python3 -m fastchat.serve.cli --model-path lmsys/vicuna-7b-v1.5 --device xpu
+```
+Vicuna-7B can run on an Intel Arc A770 16GB.
+
+#### Ascend NPU
+Install the [Ascend PyTorch Adapter](https://github.com/Ascend/pytorch). Set the CANN environment variables:
+```
+source /usr/local/Ascend/ascend-toolkit/set_env.sh
+```
+
+Use `--device npu` to enable NPU acceleration.
+```
+python3 -m fastchat.serve.cli --model-path lmsys/vicuna-7b-v1.5 --device npu
+```
+Vicuna-7B/13B can run on an Ascend NPU.
+
+#### Not Enough Memory
+If you do not have enough memory, you can enable 8-bit compression by adding `--load-8bit` to commands above.
+This can reduce memory usage by around half with slightly degraded model quality.
+It is compatible with the CPU, GPU, and Metal backend.
+
+Vicuna-13B with 8-bit compression can run on a single GPU with 16 GB of VRAM, like an Nvidia RTX 3090, RTX 4080, T4, V100 (16GB), or an AMD RX 6800 XT.
+
+```
+python3 -m fastchat.serve.cli --model-path lmsys/vicuna-7b-v1.5 --load-8bit
+```
+
+In addition to that, you can add `--cpu-offloading` to commands above to offload weights that don't fit on your GPU onto the CPU memory.
+This requires 8-bit compression to be enabled and the bitsandbytes package to be installed, which is only available on linux operating systems.
+
+#### More Platforms and Quantization
+- For AMD GPU users, please install ROCm and [the ROCm version of PyTorch](https://pytorch.org/get-started/locally/) before you install FastChat. See also this [post](https://github.com/lm-sys/FastChat/issues/104#issuecomment-1613791563).
+- FastChat supports ExLlama V2. See [docs/exllama_v2.md](/docs/exllama_v2.md).
+- FastChat supports GPTQ 4bit inference with [GPTQ-for-LLaMa](https://github.com/qwopqwop200/GPTQ-for-LLaMa). See [docs/gptq.md](/docs/gptq.md).
+- FastChat supports AWQ 4bit inference with [mit-han-lab/llm-awq](https://github.com/mit-han-lab/llm-awq). See [docs/awq.md](/docs/awq.md).
+- [MLC LLM](https://mlc.ai/mlc-llm/), backed by [TVM Unity](https://github.com/apache/tvm/tree/unity) compiler, deploys Vicuna natively on phones, consumer-class GPUs and web browsers via Vulkan, Metal, CUDA and WebGPU.
+
+#### Use models from modelscope
+For Chinese users, you can use models from www.modelscope.cn via specify the following environment variables.
+```bash
+export FASTCHAT_USE_MODELSCOPE=True
+```
+
+## Serving with Web GUI
+
+<a href="https://chat.lmsys.org"><img src="assets/screenshot_gui.png" width="70%"></a>
+
+To serve using the web UI, you need three main components: web servers that interface with users, model workers that host one or more models, and a controller to coordinate the webserver and model workers. You can learn more about the architecture [here](docs/server_arch.md).
+
+Here are the commands to follow in your terminal:
+
+#### Launch the controller
+```bash
+python3 -m fastchat.serve.controller
+```
+
+This controller manages the distributed workers.
+
+#### Launch the model worker(s)
+```bash
+python3 -m fastchat.serve.model_worker --model-path lmsys/vicuna-7b-v1.5
+```
+Wait until the process finishes loading the model and you see "Uvicorn running on ...". The model worker will register itself to the controller .
+
+To ensure that your model worker is connected to your controller properly, send a test message using the following command:
+```bash
+python3 -m fastchat.serve.test_message --model-name vicuna-7b-v1.5
+```
+You will see a short output.
+
+#### Launch the Gradio web server
+```bash
+python3 -m fastchat.serve.gradio_web_server
+```
+
+This is the user interface that users will interact with.
+
+By following these steps, you will be able to serve your models using the web UI. You can open your browser and chat with a model now.
+If the models do not show up, try to reboot the gradio web server.
+
+#### (Optional): Advanced Features, Scalability, Third Party UI
+- You can register multiple model workers to a single controller, which can be used for serving a single model with higher throughput or serving multiple models at the same time. When doing so, please allocate different GPUs and ports for different model workers.
+```
+# worker 0
+CUDA_VISIBLE_DEVICES=0 python3 -m fastchat.serve.model_worker --model-path lmsys/vicuna-7b-v1.5 --controller http://localhost:21001 --port 31000 --worker http://localhost:31000
+# worker 1
+CUDA_VISIBLE_DEVICES=1 python3 -m fastchat.serve.model_worker --model-path lmsys/fastchat-t5-3b-v1.0 --controller http://localhost:21001 --port 31001 --worker http://localhost:31001
+```
+- You can also launch a multi-tab gradio server, which includes the Chatbot Arena tabs.
+```bash
+python3 -m fastchat.serve.gradio_web_server_multi
+```
+- The default model worker based on huggingface/transformers has great compatibility but can be slow. If you want high-throughput batched serving, you can try [vLLM integration](docs/vllm_integration.md).
+- If you want to host it on your own UI or third party UI, see [Third Party UI](docs/third_party_ui.md).
+
+## API
+### OpenAI-Compatible RESTful APIs & SDK
+FastChat provides OpenAI-compatible APIs for its supported models, so you can use FastChat as a local drop-in replacement for OpenAI APIs.
+The FastChat server is compatible with both [openai-python](https://github.com/openai/openai-python) library and cURL commands.
+The REST API is capable of being executed from Google Colab free tier, as demonstrated in the [FastChat_API_GoogleColab.ipynb](https://github.com/lm-sys/FastChat/blob/main/playground/FastChat_API_GoogleColab.ipynb) notebook, available in our repository.
+See [docs/openai_api.md](docs/openai_api.md).
+
+### Hugging Face Generation APIs
+See [fastchat/serve/huggingface_api.py](fastchat/serve/huggingface_api.py).
+
+### LangChain Integration
+See [docs/langchain_integration](docs/langchain_integration.md).
+
+## Evaluation
+We use MT-bench, a set of challenging multi-turn open-ended questions to evaluate models. 
+To automate the evaluation process, we prompt strong LLMs like GPT-4 to act as judges and assess the quality of the models' responses.
+See instructions for running MT-bench at [fastchat/llm_judge](fastchat/llm_judge).
+
+MT-bench is the new recommended way to benchmark your models. If you are still looking for the old 80 questions used in the vicuna blog post, please go to [vicuna-blog-eval](https://github.com/lm-sys/vicuna-blog-eval).
+
+## Fine-tuning
+### Data
+
+Vicuna is created by fine-tuning a Llama base model using approximately 125K user-shared conversations gathered from ShareGPT.com with public APIs. To ensure data quality, we convert the HTML back to markdown and filter out some inappropriate or low-quality samples. Additionally, we divide lengthy conversations into smaller segments that fit the model's maximum context length. For detailed instructions to clean the ShareGPT data, check out [here](docs/commands/data_cleaning.md).
+
+We will not release the ShareGPT dataset. If you would like to try the fine-tuning code, you can run it with some dummy conversations in [dummy_conversation.json](data/dummy_conversation.json). You can follow the same format and plug in your own data.
+
+### Code and Hyperparameters
+Our code is based on [Stanford Alpaca](https://github.com/tatsu-lab/stanford_alpaca) with additional support for multi-turn conversations.
+We use similar hyperparameters as the Stanford Alpaca.
+
+| Hyperparameter | Global Batch Size | Learning rate | Epochs | Max length | Weight decay |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| Vicuna-13B | 128 | 2e-5 | 3 | 2048 | 0 |
+
+### Fine-tuning Vicuna-7B with Local GPUs
+
+- Install dependency
+```bash
+pip3 install -e ".[train]"
+```
+
+- You can use the following command to train Vicuna-7B with 4 x A100 (40GB). Update `--model_name_or_path` with the actual path to Llama weights and `--data_path` with the actual path to data.
+```bash
+torchrun --nproc_per_node=4 --master_port=20001 fastchat/train/train_mem.py \
+    --model_name_or_path meta-llama/Llama-2-7b-hf \
+    --data_path data/dummy_conversation.json \
+    --bf16 True \
+    --output_dir output_vicuna \
+    --num_train_epochs 3 \
+    --per_device_train_batch_size 2 \
+    --per_device_eval_batch_size 2 \
+    --gradient_accumulation_steps 16 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 1200 \
+    --save_total_limit 10 \
+    --learning_rate 2e-5 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --fsdp "full_shard auto_wrap" \
+    --fsdp_transformer_layer_cls_to_wrap 'LlamaDecoderLayer' \
+    --tf32 True \
+    --model_max_length 2048 \
+    --gradient_checkpointing True \
+    --lazy_preprocess True
+```
+
+Tips:
+- If you are using V100 which is not supported by FlashAttention, you can use the [memory-efficient attention](https://arxiv.org/abs/2112.05682) implemented in [xFormers](https://github.com/facebookresearch/xformers). Install xformers and replace `fastchat/train/train_mem.py` above with [fastchat/train/train_xformers.py](fastchat/train/train_xformers.py).
+- If you meet out-of-memory due to "FSDP Warning: When using FSDP, it is efficient and recommended... ", see solutions [here](https://github.com/huggingface/transformers/issues/24724#issuecomment-1645189539).
+- If you meet out-of-memory during model saving, see solutions [here](https://github.com/pytorch/pytorch/issues/98823).
+- To turn on logging to popular experiment tracking tools such as Tensorboard, MLFlow or Weights & Biases, use the `report_to` argument, e.g. pass `--report_to wandb` to turn on logging to Weights & Biases.
+
+### Other models, platforms and LoRA support
+More instructions to train other models (e.g., FastChat-T5) and use LoRA are in [docs/training.md](docs/training.md).
+
+### Fine-tuning on Any Cloud with SkyPilot
+[SkyPilot](https://github.com/skypilot-org/skypilot) is a framework built by UC Berkeley for easily and cost effectively running ML workloads on any cloud (AWS, GCP, Azure, Lambda, etc.).
+Find SkyPilot documentation [here](https://github.com/skypilot-org/skypilot/tree/master/llm/vicuna) on using managed spot instances to train Vicuna and save on your cloud costs.
+
+## Citation
+The code (training, serving, and evaluation) in this repository is mostly developed for or derived from the paper below.
+Please cite it if you find the repository helpful.
+
+```
+@misc{zheng2023judging,
+      title={Judging LLM-as-a-judge with MT-Bench and Chatbot Arena},
+      author={Lianmin Zheng and Wei-Lin Chiang and Ying Sheng and Siyuan Zhuang and Zhanghao Wu and Yonghao Zhuang and Zi Lin and Zhuohan Li and Dacheng Li and Eric. P Xing and Hao Zhang and Joseph E. Gonzalez and Ion Stoica},
+      year={2023},
+      eprint={2306.05685},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+```
+
+We are also planning to add more of our research to this repository.

docker/Dockerfile

@@ -0,0 +1,7 @@
+FROM nvidia/cuda:12.2.0-runtime-ubuntu20.04
+
+RUN apt-get update -y && apt-get install -y python3.9 python3.9-distutils curl
+RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+RUN python3.9 get-pip.py
+RUN pip3 install fschat
+RUN pip3 install fschat[model_worker,webui] pydantic==1.10.13

docker/docker-compose.yml

@@ -0,0 +1,36 @@
+version: "3.9"
+
+services:
+  fastchat-controller:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: fastchat:latest
+    ports:
+      - "21001:21001"
+    entrypoint: ["python3.9", "-m", "fastchat.serve.controller", "--host", "0.0.0.0", "--port", "21001"]
+  fastchat-model-worker:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      - huggingface:/root/.cache/huggingface
+    image: fastchat:latest
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: [gpu]
+    entrypoint: ["python3.9", "-m", "fastchat.serve.model_worker", "--model-names", "${FASTCHAT_WORKER_MODEL_NAMES:-vicuna-7b-v1.5}", "--model-path", "${FASTCHAT_WORKER_MODEL_PATH:-lmsys/vicuna-7b-v1.5}", "--worker-address", "http://fastchat-model-worker:21002", "--controller-address", "http://fastchat-controller:21001", "--host", "0.0.0.0", "--port", "21002"]
+  fastchat-api-server:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: fastchat:latest
+    ports:
+      - "8000:8000"
+    entrypoint: ["python3.9", "-m", "fastchat.serve.openai_api_server", "--controller-address", "http://fastchat-controller:21001", "--host", "0.0.0.0", "--port", "8000"]
+volumes:
+  huggingface:

docs/arena.md

@@ -0,0 +1,15 @@
+# Chatbot Arena
+Chatbot Arena is an LLM benchmark platform featuring anonymous, randomized battles, available at https://chat.lmsys.org.
+We invite the entire community to join this benchmarking effort by contributing your votes and models.
+
+## How to add a new model
+If you want to see a specific model in the arena, you can follow the methods below.
+
+### Method 1: Hosted by 3rd party API providers or yourself
+If you have a model hosted by a 3rd party API provider or yourself, please give us the access to an API endpoint.
+  - We prefer OpenAI-compatible APIs, so we can reuse our [code](https://github.com/lm-sys/FastChat/blob/main/fastchat/serve/api_provider.py) for calling OpenAI models.
+  - If you have your own API protocol, please follow the [instructions](model_support.md) to add them. Contribute your code by sending a pull request.
+
+### Method 2: Hosted by LMSYS
+1. Contribute the code to support this model in FastChat by submitting a pull request. See [instructions](model_support.md).
+2. After the model is supported, we will try to schedule some compute resources to host the model in the arena. However, due to the limited resources we have, we may not be able to serve every model. We will select the models based on popularity, quality, diversity, and other factors.

docs/awq.md

@@ -0,0 +1,71 @@
+# AWQ 4bit Inference
+
+We integrated [AWQ](https://github.com/mit-han-lab/llm-awq) into FastChat to provide **efficient and accurate** 4bit LLM inference.
+
+## Install AWQ
+
+Setup environment (please refer to [this link](https://github.com/mit-han-lab/llm-awq#install) for more details):
+```bash
+conda create -n fastchat-awq python=3.10 -y
+conda activate fastchat-awq
+# cd /path/to/FastChat
+pip install --upgrade pip    # enable PEP 660 support
+pip install -e .             # install fastchat
+
+git clone https://github.com/mit-han-lab/llm-awq repositories/llm-awq
+cd repositories/llm-awq
+pip install -e .             # install awq package
+
+cd awq/kernels				
+python setup.py install	     # install awq CUDA kernels
+```
+
+## Chat with the CLI
+
+```bash
+# Download quantized model from huggingface
+# Make sure you have git-lfs installed (https://git-lfs.com)
+git lfs install
+git clone https://huggingface.co/mit-han-lab/vicuna-7b-v1.3-4bit-g128-awq
+
+# You can specify which quantized model to use by setting --awq-ckpt
+python3 -m fastchat.serve.cli \
+    --model-path models/vicuna-7b-v1.3-4bit-g128-awq \
+    --awq-wbits 4 \
+    --awq-groupsize 128 
+```
+
+## Benchmark
+
+* Through **4-bit weight quantization**, AWQ helps to run larger language models within the device memory restriction and prominently accelerates token generation. All benchmarks are done with group_size 128. 
+
+* Benchmark on NVIDIA RTX A6000:
+
+  | Model           | Bits | Max Memory (MiB) | Speed (ms/token) | AWQ Speedup |
+  | --------------- | ---- | ---------------- | ---------------- | ----------- |
+  | vicuna-7b       | 16   | 13543            | 26.06            | /           |
+  | vicuna-7b       | 4    | 5547             | 12.43            | 2.1x        |
+  | llama2-7b-chat  | 16   | 13543            | 27.14            | /           |
+  | llama2-7b-chat  | 4    | 5547             | 12.44            | 2.2x        |
+  | vicuna-13b      | 16   | 25647            | 44.91            | /           |
+  | vicuna-13b      | 4    | 9355             | 17.30            | 2.6x        |
+  | llama2-13b-chat | 16   | 25647            | 47.28            | /           |
+  | llama2-13b-chat | 4    | 9355             | 20.28            | 2.3x        |
+
+* NVIDIA RTX 4090:
+
+  | Model           | AWQ 4bit Speed (ms/token) | FP16 Speed (ms/token) | AWQ Speedup |
+  | --------------- | ------------------------- | --------------------- | ----------- |
+  | vicuna-7b       | 8.61                      | 19.09                 | 2.2x        |
+  | llama2-7b-chat  | 8.66                      | 19.97                 | 2.3x        |
+  | vicuna-13b      | 12.17                     | OOM                   | /           |
+  | llama2-13b-chat | 13.54                     | OOM                   | /           |
+
+* NVIDIA Jetson Orin:
+
+  | Model           | AWQ 4bit Speed (ms/token) | FP16 Speed (ms/token) | AWQ Speedup |
+  | --------------- | ------------------------- | --------------------- | ----------- |
+  | vicuna-7b       | 65.34                     | 93.12                 | 1.4x        |
+  | llama2-7b-chat  | 75.11                     | 104.71                | 1.4x        |
+  | vicuna-13b      | 115.40                    | OOM                   | /           |
+  | llama2-13b-chat | 136.81                    | OOM                   | /           |

docs/commands/conv_release.md

@@ -0,0 +1,38 @@
+## Chatbot Arena Conversations
+
+1. Gather battles
+```
+python3 clean_battle_data.py --max-num 10 --mode conv_release
+```
+
+2. Tag OpenAI moderation
+```
+python3 tag_openai_moderation.py --in clean_battle_conv_20230814.json
+```
+
+3. Clean PII
+
+4. Filter additional blocked words
+
+```
+python3 filter_bad_conv.py --in clean_battle_conv_20230630_tagged_v1_pii.json
+```
+
+5. Add additional toxicity tag
+
+
+## All Conversations
+
+1. Gather chats
+```
+python3 clean_chat_data.py
+```
+
+2. Sample
+```
+python3 conv_release_scripts/sample.py
+```
+
+
+## Prompt distribution
+

docs/commands/data_cleaning.md

@@ -0,0 +1,19 @@
+## Data cleaning
+
+## Requirements
+```
+pip3 install bs4 markdownify
+pip3 install polyglot pyicu pycld2
+```
+
+## Steps
+```
+# Convert html to markdown
+python3 -m fastchat.data.clean_sharegpt --in sharegpt_html.json --out sharegpt_clean.json
+
+# Keep or remove specific languages
+python3 -m fastchat.data.optional_clean --in sharegpt_clean.json --out sharegpt_clean_lang.json --skip-lang SOME_LANGUAGE_CODE
+
+# Split long conversations
+python3 -m fastchat.data.split_long_conversation --in sharegpt_clean_lang.json --out sharegpt_clean_lang_split.json --model-name /home/ubuntu/model_weights/llama-7b/
+```

docs/commands/leaderboard.md

@@ -0,0 +1,37 @@
+### Get logs
+```
+gsutil -m rsync -r gs://fastchat_logs ~/fastchat_logs/
+```
+
+### Clean battle data
+```
+cd ~/FastChat/fastchat/serve/monitor
+python3 clean_battle_data.py
+```
+
+### Run Elo analysis
+```
+python3 elo_analysis.py --clean-battle-file clean_battle_20230905.json
+```
+
+### Copy files to HF space
+1. update plots
+```
+scp atlas:/data/lmzheng/FastChat/fastchat/serve/monitor/elo_results_20230905.pkl .
+```
+
+2. update table
+```
+wget https://huggingface.co/spaces/lmsys/chatbot-arena-leaderboard/raw/main/leaderboard_table_20230905.csv
+```
+
+### Update files on webserver
+```
+DATE=20231002
+
+rm -rf elo_results.pkl leaderboard_table.csv
+wget https://huggingface.co/spaces/lmsys/chatbot-arena-leaderboard/resolve/main/elo_results_$DATE.pkl
+wget https://huggingface.co/spaces/lmsys/chatbot-arena-leaderboard/resolve/main/leaderboard_table_$DATE.csv
+ln -s leaderboard_table_$DATE.csv leaderboard_table.csv
+ln -s elo_results_$DATE.pkl elo_results.pkl
+```

docs/commands/local_cluster.md

@@ -0,0 +1,38 @@
+### Local GPU cluster
+node-01
+```
+python3 -m fastchat.serve.controller --host 0.0.0.0 --port 10002
+
+CUDA_VISIBLE_DEVICES=0 python3 -m fastchat.serve.vllm_worker --model-path lmsys/vicuna-13b-v1.5 --model-name vicuna-13b --controller http://node-01:10002 --host 0.0.0.0 --port 31000 --worker-address http://$(hostname):31000
+CUDA_VISIBLE_DEVICES=1 python3 -m fastchat.serve.vllm_worker --model-path lmsys/vicuna-13b-v1.5 --model-name vicuna-13b --controller http://node-01:10002 --host 0.0.0.0 --port 31001 --worker-address http://$(hostname):31001
+
+CUDA_VISIBLE_DEVICES=2,3 ray start --head
+python3 -m fastchat.serve.vllm_worker --model-path lmsys/vicuna-33b-v1.3 --model-name vicuna-33b --controller http://node-01:10002 --host 0.0.0.0 --port 31002 --worker-address http://$(hostname):31002 --num-gpus 2
+```
+
+node-02
+```
+CUDA_VISIBLE_DEVICES=0 python3 -m fastchat.serve.vllm_worker --model-path meta-llama/Llama-2-13b-chat-hf --model-name llama-2-13b-chat --controller http://node-01:10002 --host 0.0.0.0 --port 31000 --worker-address http://$(hostname):31000 --tokenizer meta-llama/Llama-2-7b-chat-hf
+CUDA_VISIBLE_DEVICES=1 python3 -m fastchat.serve.vllm_worker --model-path meta-llama/Llama-2-13b-chat-hf --model-name llama-2-13b-chat --controller http://node-01:10002 --host 0.0.0.0 --port 31001 --worker-address http://$(hostname):31001 --tokenizer meta-llama/Llama-2-7b-chat-hf
+CUDA_VISIBLE_DEVICES=2 python3 -m fastchat.serve.vllm_worker --model-path meta-llama/Llama-2-7b-chat-hf --model-name llama-2-7b-chat --controller http://node-01:10002 --host 0.0.0.0 --port 31002 --worker-address http://$(hostname):31002 --tokenizer meta-llama/Llama-2-7b-chat-hf
+CUDA_VISIBLE_DEVICES=3 python3 -m fastchat.serve.vllm_worker --model-path WizardLM/WizardLM-13B-V1.1 --model-name wizardlm-13b  --controller http://node-01:10002 --host 0.0.0.0 --port 31003 --worker-address http://$(hostname):31003
+```
+
+node-03
+```
+python3 -m fastchat.serve.vllm_worker --model-path mosaicml/mpt-30b-chat --controller http://node-01:10002 --host 0.0.0.0 --port 31000 --worker-address http://$(hostname):31000 --num-gpus 2
+python3 -m fastchat.serve.vllm_worker --model-path timdettmers/guanaco-33b-merged --model-name guanaco-33b  --controller http://node-01:10002 --host 0.0.0.0 --port 31002 --worker-address http://$(hostname):31002 --num-gpus 2 --tokenizer hf-internal-testing/llama-tokenizer
+```
+
+node-04
+```
+CUDA_VISIBLE_DEVICES=0 python3 -m fastchat.serve.multi_model_worker --model-path ~/model_weights/RWKV-4-Raven-14B-v12-Eng98%25-Other2%25-20230523-ctx8192.pth --model-name RWKV-4-Raven-14B --model-path lmsys/fastchat-t5-3b-v1.0 --model-name fastchat-t5-3b --controller http://node-01:10002 --host 0.0.0.0 --port 31000 --worker http://$(hostname):31000 --limit 4
+CUDA_VISIBLE_DEVICES=1 python3 -m fastchat.serve.multi_model_worker --model-path OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5 --model-name oasst-pythia-12b --model-path mosaicml/mpt-7b-chat --model-name mpt-7b-chat --controller http://node-01:10002 --host 0.0.0.0 --port 31001 --worker http://$(hostname):31001 --limit 4
+CUDA_VISIBLE_DEVICES=2 python3 -m fastchat.serve.multi_model_worker --model-path lmsys/vicuna-7b-v1.5 --model-name vicuna-7b --model-path THUDM/chatglm-6b --model-name chatglm-6b --controller http://node-01:10002 --host 0.0.0.0 --port 31002 --worker http://$(hostname):31002 --limit 4
+CUDA_VISIBLE_DEVICES=3 python3 -m fastchat.serve.vllm_worker --model-path ~/model_weights/alpaca-13b  --controller http://node-01:10002 --host 0.0.0.0 --port 31003 --worker-address http://$(hostname):31003
+```
+
+test
+```
+python3 -m fastchat.serve.test_message --model vicuna-13b --controller http://localhost:10002
+```

docs/commands/pypi.md

@@ -0,0 +1,11 @@
+### Requirement
+```
+python3 -m pip install twine
+python3 -m pip install --upgrade pip
+pip3 install build
+```
+
+### Upload
+```
+bash scripts/upload_pypi.sh
+```

docs/commands/webserver.md

@@ -0,0 +1,94 @@
+### Install
+```
+sudo apt update
+sudo apt install tmux htop
+
+wget https://repo.anaconda.com/archive/Anaconda3-2022.10-Linux-x86_64.sh
+bash Anaconda3-2022.10-Linux-x86_64.sh
+
+conda create -n fastchat python=3.9
+conda activate fastchat
+
+git clone https://github.com/lm-sys/FastChat.git
+cd FastChat
+pip3 install -e .
+```
+
+
+### Launch servers
+```
+cd fastchat_logs/controller
+python3 -m fastchat.serve.controller --host 0.0.0.0 --port 21001
+python3 -m fastchat.serve.register_worker --controller http://localhost:21001 --worker-name https://
+python3 -m fastchat.serve.test_message --model vicuna-13b --controller http://localhost:21001
+
+cd fastchat_logs/server0
+
+python3 -m fastchat.serve.huggingface_api_worker --model-info-file ~/elo_results/register_hf_api_models.json
+
+export OPENAI_API_KEY=
+export ANTHROPIC_API_KEY=
+export GCP_PROJECT_ID=
+
+python3 -m fastchat.serve.gradio_web_server_multi --controller http://localhost:21001 --concurrency 50 --add-chatgpt --add-claude --add-palm --elo ~/elo_results/elo_results.pkl --leaderboard-table-file ~/elo_results/leaderboard_table.csv --register ~/elo_results/register_oai_models.json --show-terms
+
+python3 backup_logs.py
+```
+
+
+### Check the launch time
+```
+for i in $(seq 0 11); do cat fastchat_logs/server$i/gradio_web_server.log | grep "Running on local URL" | tail -n 1; done
+```
+
+
+### Increase the limit of max open files
+One process (do not need reboot)
+```
+sudo prlimit --nofile=1048576:1048576 --pid=$id
+
+for id in $(ps -ef | grep gradio_web_server | awk '{print $2}'); do echo $id; prlimit --nofile=1048576:1048576 --pid=$id; done
+```
+
+System (need reboot): Add the lines below to `/etc/security/limits.conf`
+```
+* hard nofile 65535
+* soft nofile 65535
+```
+
+
+### Gradio edit  (3.35.2)
+1. gtag and canvas
+```
+vim /home/vicuna/anaconda3/envs/fastchat/lib/python3.9/site-packages/gradio/templates/frontend/index.html
+```
+
+```
+<!-- Google tag (gtag.js) -->
+<script async src="https://www.googletagmanager.com/gtag/js?id=G-K6D24EE9ED"></script><script>
+  window.dataLayer = window.dataLayer || [];
+  function gtag(){dataLayer.push(arguments);}
+  gtag('js', new Date());
+  gtag('config', 'G-K6D24EE9ED');
+  window.__gradio_mode__ = "app";
+</script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/html2canvas/1.4.1/html2canvas.min.js"></script>
+```
+
+2. deprecation warnings
+```
+vim /home/vicuna/anaconda3/envs/fastchat/lib/python3.9/site-packages/gradio/deprecation.py
+```
+
+```
+def check_deprecated_parameters(
+```
+
+3. Loading
+```
+vim /home/vicuna/anaconda3/envs/fastchat/lib/python3.9/site-packages/gradio/templates/frontend/assets/index-188ef5e8.js
+```
+
+```
+%s/"Loading..."/"Loading...(Please refresh if it takes more than 30 seconds)"/g
+```

docs/dataset_release.md

@@ -0,0 +1,6 @@
+## Datasets
+We release the following datasets based on our projects and websites.
+
+- [LMSYS-Chat-1M: A Large-Scale Real-World LLM Conversation Dataset](https://huggingface.co/datasets/lmsys/lmsys-chat-1m)
+- [Chatbot Arena Conversation Dataset](https://huggingface.co/datasets/lmsys/chatbot_arena_conversations)
+- [MT-bench Human Annotation Dataset](https://huggingface.co/datasets/lmsys/mt_bench_human_judgments)

docs/exllama_v2.md

@@ -0,0 +1,63 @@
+# ExllamaV2 GPTQ Inference Framework
+
+Integrated [ExllamaV2](https://github.com/turboderp/exllamav2) customized kernel into Fastchat to provide **Faster** GPTQ inference speed.
+
+**Note: Exllama not yet support embedding REST API.**
+
+## Install ExllamaV2
+
+Setup environment (please refer to [this link](https://github.com/turboderp/exllamav2#how-to) for more details):
+
+```bash
+git clone https://github.com/turboderp/exllamav2
+cd exllamav2
+pip install -e .
+```
+
+Chat with the CLI:
+```bash
+python3 -m fastchat.serve.cli \
+    --model-path models/vicuna-7B-1.1-GPTQ-4bit-128g \
+    --enable-exllama
+```
+
+Start model worker:
+```bash
+# Download quantized model from huggingface
+# Make sure you have git-lfs installed (https://git-lfs.com)
+git lfs install
+git clone https://huggingface.co/TheBloke/vicuna-7B-1.1-GPTQ-4bit-128g models/vicuna-7B-1.1-GPTQ-4bit-128g
+
+# Load model with default configuration (max sequence length 4096, no GPU split setting).
+python3 -m fastchat.serve.model_worker \
+    --model-path models/vicuna-7B-1.1-GPTQ-4bit-128g \
+    --enable-exllama
+
+#Load model with max sequence length 2048, allocate 18 GB to CUDA:0 and 24 GB to CUDA:1.
+python3 -m fastchat.serve.model_worker \
+    --model-path models/vicuna-7B-1.1-GPTQ-4bit-128g \
+    --enable-exllama \
+    --exllama-max-seq-len 2048 \
+    --exllama-gpu-split 18,24
+```
+
+`--exllama-cache-8bit` can be used to enable 8-bit caching with exllama and save some VRAM.
+
+## Performance 
+
+Reference: https://github.com/turboderp/exllamav2#performance
+
+
+| Model      | Mode         | Size  | grpsz | act | V1: 3090Ti | V1: 4090 | V2: 3090Ti | V2: 4090    |
+|------------|--------------|-------|-------|-----|------------|----------|------------|-------------|
+| Llama      | GPTQ         | 7B    | 128   | no  | 143 t/s    | 173 t/s  | 175 t/s    | **195** t/s |
+| Llama      | GPTQ         | 13B   | 128   | no  | 84 t/s     | 102 t/s  | 105 t/s    | **110** t/s |
+| Llama      | GPTQ         | 33B   | 128   | yes | 37 t/s     | 45 t/s   | 45 t/s     | **48** t/s  |
+| OpenLlama  | GPTQ         | 3B    | 128   | yes | 194 t/s    | 226 t/s  | 295 t/s    | **321** t/s |
+| CodeLlama  | EXL2 4.0 bpw | 34B   | -     | -   | -          | -        | 42 t/s     | **48** t/s  |
+| Llama2     | EXL2 3.0 bpw | 7B    | -     | -   | -          | -        | 195 t/s    | **224** t/s |
+| Llama2     | EXL2 4.0 bpw | 7B    | -     | -   | -          | -        | 164 t/s    | **197** t/s |
+| Llama2     | EXL2 5.0 bpw | 7B    | -     | -   | -          | -        | 144 t/s    | **160** t/s |
+| Llama2     | EXL2 2.5 bpw | 70B   | -     | -   | -          | -        | 30 t/s     | **35** t/s  |
+| TinyLlama  | EXL2 3.0 bpw | 1.1B  | -     | -   | -          | -        | 536 t/s    | **635** t/s |
+| TinyLlama  | EXL2 4.0 bpw | 1.1B  | -     | -   | -          | -        | 509 t/s    | **590** t/s |

docs/gptq.md

@@ -0,0 +1,59 @@
+# GPTQ 4bit Inference
+
+Support GPTQ 4bit inference with [GPTQ-for-LLaMa](https://github.com/qwopqwop200/GPTQ-for-LLaMa).
+
+1. Window user: use the `old-cuda` branch.
+2. Linux user: recommend the `fastest-inference-4bit` branch.
+
+## Install
+
+Setup environment:
+```bash
+# cd /path/to/FastChat
+git clone https://github.com/qwopqwop200/GPTQ-for-LLaMa.git repositories/GPTQ-for-LLaMa
+cd repositories/GPTQ-for-LLaMa
+# Window's user should use the `old-cuda` branch
+git switch fastest-inference-4bit
+# Install `quant-cuda` package in FastChat's virtualenv
+python3 setup_cuda.py install
+pip3 install texttable
+```
+
+Chat with the CLI:
+```bash
+python3 -m fastchat.serve.cli \
+    --model-path models/vicuna-7B-1.1-GPTQ-4bit-128g \
+    --gptq-wbits 4 \
+    --gptq-groupsize 128
+```
+
+Start model worker:
+```bash
+# Download quantized model from huggingface
+# Make sure you have git-lfs installed (https://git-lfs.com)
+git lfs install
+git clone https://huggingface.co/TheBloke/vicuna-7B-1.1-GPTQ-4bit-128g models/vicuna-7B-1.1-GPTQ-4bit-128g
+
+python3 -m fastchat.serve.model_worker \
+    --model-path models/vicuna-7B-1.1-GPTQ-4bit-128g \
+    --gptq-wbits 4 \
+    --gptq-groupsize 128
+
+# You can specify which quantized model to use
+python3 -m fastchat.serve.model_worker \
+    --model-path models/vicuna-7B-1.1-GPTQ-4bit-128g \
+    --gptq-ckpt models/vicuna-7B-1.1-GPTQ-4bit-128g/vicuna-7B-1.1-GPTQ-4bit-128g.safetensors \
+    --gptq-wbits 4 \
+    --gptq-groupsize 128 \
+    --gptq-act-order
+```
+
+## Benchmark
+
+| LLaMA-13B | branch                 | Bits | group-size | memory(MiB) | PPL(c4) | Median(s/token) | act-order | speed up |
+| --------- | ---------------------- | ---- | ---------- | ----------- | ------- | --------------- | --------- | -------- |
+| FP16      | fastest-inference-4bit | 16   | -          | 26634       | 6.96    | 0.0383          | -         | 1x       |
+| GPTQ      | triton                 | 4    | 128        | 8590        | 6.97    | 0.0551          | -         | 0.69x    |
+| GPTQ      | fastest-inference-4bit | 4    | 128        | 8699        | 6.97    | 0.0429          | true      | 0.89x    |
+| GPTQ      | fastest-inference-4bit | 4    | 128        | 8699        | 7.03    | 0.0287          | false     | 1.33x    |
+| GPTQ      | fastest-inference-4bit | 4    | -1         | 8448        | 7.12    | 0.0284          | false     | 1.44x    |

docs/langchain_integration.md

@@ -0,0 +1,90 @@
+# Local LangChain with FastChat
+
+[LangChain](https://python.langchain.com/en/latest/index.html) is a library that facilitates the development of applications by leveraging large language models (LLMs) and enabling their composition with other sources of computation or knowledge.
+FastChat's OpenAI-compatible [API server](openai_api.md) enables using LangChain with open models seamlessly.
+
+## Launch RESTful API Server
+
+Here are the steps to launch a local OpenAI API server for LangChain.
+
+First, launch the controller
+
+```bash
+python3 -m fastchat.serve.controller
+```
+
+LangChain uses OpenAI model names by default, so we need to assign some faux OpenAI model names to our local model.
+Here, we use Vicuna as an example and use it for three endpoints: chat completion, completion, and embedding.
+`--model-path` can be a local folder or a Hugging Face repo name.
+See a full list of supported models [here](../README.md#supported-models).
+
+```bash
+python3 -m fastchat.serve.model_worker --model-names "gpt-3.5-turbo,text-davinci-003,text-embedding-ada-002" --model-path lmsys/vicuna-7b-v1.5
+```
+
+Finally, launch the RESTful API server
+
+```bash
+python3 -m fastchat.serve.openai_api_server --host localhost --port 8000
+```
+
+## Set OpenAI Environment
+
+You can set your environment with the following commands.
+
+Set OpenAI base url
+
+```bash
+export OPENAI_API_BASE=http://localhost:8000/v1
+```
+
+Set OpenAI API key
+
+```bash
+export OPENAI_API_KEY=EMPTY
+```
+
+If you meet the following OOM error while creating embeddings, please set a smaller batch size by using environment variables.
+
+~~~bash
+openai.error.APIError: Invalid response object from API: '{"object":"error","message":"**NETWORK ERROR DUE TO HIGH TRAFFIC. PLEASE REGENERATE OR REFRESH THIS PAGE.**\\n\\n(CUDA out of memory. Tried to allocate xxx MiB (GPU 0; xxx GiB total capacity; xxx GiB already allocated; xxx MiB free; xxx GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF)","code":50002}' (HTTP response code was 400)
+~~~
+
+You can try `export FASTCHAT_WORKER_API_EMBEDDING_BATCH_SIZE=1`.
+
+## Try local LangChain
+
+Here is a question answerting example.
+
+Download a text file.
+
+```bash
+wget https://raw.githubusercontent.com/hwchase17/langchain/v0.0.200/docs/modules/state_of_the_union.txt
+```
+
+Run LangChain.
+
+~~~py
+from langchain.chat_models import ChatOpenAI
+from langchain.document_loaders import TextLoader
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.indexes import VectorstoreIndexCreator
+
+embedding = OpenAIEmbeddings(model="text-embedding-ada-002")
+loader = TextLoader("state_of_the_union.txt")
+index = VectorstoreIndexCreator(embedding=embedding).from_loaders([loader])
+llm = ChatOpenAI(model="gpt-3.5-turbo")
+
+questions = [
+    "Who is the speaker",
+    "What did the president say about Ketanji Brown Jackson",
+    "What are the threats to America",
+    "Who are mentioned in the speech",
+    "Who is the vice president",
+    "How many projects were announced",
+]
+
+for query in questions:
+    print("Query:", query)
+    print("Answer:", index.query(query, llm=llm))
+~~~

docs/lightllm_integration.md

@@ -0,0 +1,18 @@
+# LightLLM Integration
+You can use [LightLLM](https://github.com/ModelTC/lightllm) as an optimized worker implementation in FastChat.
+It offers advanced continuous batching and a much higher (~10x) throughput.
+See the supported models [here](https://github.com/ModelTC/lightllm?tab=readme-ov-file#supported-model-list).
+
+## Instructions
+1. Please refer to the [Get started](https://github.com/ModelTC/lightllm?tab=readme-ov-file#get-started) to install LightLLM. Or use [Pre-built image](https://github.com/ModelTC/lightllm?tab=readme-ov-file#container)
+
+2. When you launch a model worker, replace the normal worker (`fastchat.serve.model_worker`) with the LightLLM worker (`fastchat.serve.lightllm_worker`). All other commands such as controller, gradio web server, and OpenAI API server are kept the same. Refer to [--max_total_token_num](https://github.com/ModelTC/lightllm/blob/4a9824b6b248f4561584b8a48ae126a0c8f5b000/docs/ApiServerArgs.md?plain=1#L23) to understand how to calculate the `--max_total_token_num` argument.
+   ```
+   python3 -m fastchat.serve.lightllm_worker --model-path lmsys/vicuna-7b-v1.5 --tokenizer_mode "auto" --max_total_token_num 154000
+   ```
+
+   If you what to use quantized weight and kv cache for inference, try
+
+   ```
+   python3 -m fastchat.serve.lightllm_worker --model-path lmsys/vicuna-7b-v1.5 --tokenizer_mode "auto" --max_total_token_num 154000 --mode triton_int8weight triton_int8kv
+   ```

docs/mlx_integration.md

@@ -0,0 +1,23 @@
+# Apple MLX Integration
+
+You can use [Apple MLX](https://github.com/ml-explore/mlx) as an optimized worker implementation in FastChat.
+
+It runs models efficiently on Apple Silicon
+
+See the supported models [here](https://github.com/ml-explore/mlx-examples/tree/main/llms#supported-models).
+
+Note that for Apple Silicon Macs with less memory, smaller models (or quantized models) are recommended.
+
+## Instructions
+
+1. Install MLX.
+
+   ```
+   pip install "mlx-lm>=0.0.6"
+   ```
+
+2. When you launch a model worker, replace the normal worker (`fastchat.serve.model_worker`) with the MLX worker (`fastchat.serve.mlx_worker`). Remember to launch a model worker after you have launched the controller ([instructions](../README.md))
+
+   ```
+   python3 -m fastchat.serve.mlx_worker --model-path TinyLlama/TinyLlama-1.1B-Chat-v1.0
+   ```

docs/model_support.md

@@ -0,0 +1,130 @@
+# Model Support
+This document describes how to support a new model in FastChat.
+
+## Content
+- [Local Models](#local-models)
+- [API-Based Models](#api-based-models)
+
+## Local Models
+To support a new local model in FastChat, you need to correctly handle its prompt template and model loading.
+The goal is to make the following command run with the correct prompts.
+
+```
+python3 -m fastchat.serve.cli --model [YOUR_MODEL_PATH]
+```
+
+You can run this example command to learn the code logic.
+
+```
+python3 -m fastchat.serve.cli --model lmsys/vicuna-7b-v1.5
+```
+
+You can add `--debug` to see the actual prompt sent to the model.
+
+### Steps
+
+FastChat uses the `Conversation` class to handle prompt templates and `BaseModelAdapter` class to handle model loading.
+
+1. Implement a conversation template for the new model at [fastchat/conversation.py](https://github.com/lm-sys/FastChat/blob/main/fastchat/conversation.py). You can follow existing examples and use `register_conv_template` to add a new one. Please also add a link to the official reference code if possible.
+2. Implement a model adapter for the new model at [fastchat/model/model_adapter.py](https://github.com/lm-sys/FastChat/blob/main/fastchat/model/model_adapter.py). You can follow existing examples and use `register_model_adapter` to add a new one.
+3. (Optional) add the model name to the "Supported models" [section](#supported-models) above and add more information in [fastchat/model/model_registry.py](https://github.com/lm-sys/FastChat/blob/main/fastchat/model/model_registry.py).
+
+After these steps, the new model should be compatible with most FastChat features, such as CLI, web UI, model worker, and OpenAI-compatible API server. Please do some testing with these features as well.
+
+### Supported models
+
+- [meta-llama/Llama-2-7b-chat-hf](https://huggingface.co/meta-llama/Llama-2-7b-chat-hf)
+  - example: `python3 -m fastchat.serve.cli --model-path meta-llama/Llama-2-7b-chat-hf`
+- Vicuna, Alpaca, LLaMA, Koala
+  - example: `python3 -m fastchat.serve.cli --model-path lmsys/vicuna-7b-v1.5`
+- [allenai/tulu-2-dpo-7b](https://huggingface.co/allenai/tulu-2-dpo-7b)
+- [BAAI/AquilaChat-7B](https://huggingface.co/BAAI/AquilaChat-7B)
+- [BAAI/AquilaChat2-7B](https://huggingface.co/BAAI/AquilaChat2-7B)
+- [BAAI/AquilaChat2-34B](https://huggingface.co/BAAI/AquilaChat2-34B)
+- [BAAI/bge-large-en](https://huggingface.co/BAAI/bge-large-en#using-huggingface-transformers)
+- [argilla/notus-7b-v1](https://huggingface.co/argilla/notus-7b-v1)
+- [baichuan-inc/baichuan-7B](https://huggingface.co/baichuan-inc/baichuan-7B)
+- [BlinkDL/RWKV-4-Raven](https://huggingface.co/BlinkDL/rwkv-4-raven)
+  - example: `python3 -m fastchat.serve.cli --model-path ~/model_weights/RWKV-4-Raven-7B-v11x-Eng99%-Other1%-20230429-ctx8192.pth`
+- [bofenghuang/vigogne-2-7b-instruct](https://huggingface.co/bofenghuang/vigogne-2-7b-instruct)
+- [bofenghuang/vigogne-2-7b-chat](https://huggingface.co/bofenghuang/vigogne-2-7b-chat)
+- [camel-ai/CAMEL-13B-Combined-Data](https://huggingface.co/camel-ai/CAMEL-13B-Combined-Data)
+- [codellama/CodeLlama-7b-Instruct-hf](https://huggingface.co/codellama/CodeLlama-7b-Instruct-hf)
+- [databricks/dolly-v2-12b](https://huggingface.co/databricks/dolly-v2-12b)
+- [deepseek-ai/deepseek-llm-67b-chat](https://huggingface.co/deepseek-ai/deepseek-llm-67b-chat)
+- [deepseek-ai/deepseek-coder-33b-instruct](https://huggingface.co/deepseek-ai/deepseek-coder-33b-instruct)
+- [FlagAlpha/Llama2-Chinese-13b-Chat](https://huggingface.co/FlagAlpha/Llama2-Chinese-13b-Chat)
+- [FreedomIntelligence/phoenix-inst-chat-7b](https://huggingface.co/FreedomIntelligence/phoenix-inst-chat-7b)
+- [FreedomIntelligence/ReaLM-7b-v1](https://huggingface.co/FreedomIntelligence/Realm-7b)
+- [h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b](https://huggingface.co/h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b)
+- [HuggingFaceH4/starchat-beta](https://huggingface.co/HuggingFaceH4/starchat-beta)
+- [HuggingFaceH4/zephyr-7b-alpha](https://huggingface.co/HuggingFaceH4/zephyr-7b-alpha)
+- [internlm/internlm-chat-7b](https://huggingface.co/internlm/internlm-chat-7b)
+- [IEITYuan/Yuan2-2B/51B/102B-hf](https://huggingface.co/IEITYuan)
+- [lcw99/polyglot-ko-12.8b-chang-instruct-chat](https://huggingface.co/lcw99/polyglot-ko-12.8b-chang-instruct-chat)
+- [lmsys/fastchat-t5-3b-v1.0](https://huggingface.co/lmsys/fastchat-t5)
+- [meta-math/MetaMath-7B-V1.0](https://huggingface.co/meta-math/MetaMath-7B-V1.0)
+- [Microsoft/Orca-2-7b](https://huggingface.co/microsoft/Orca-2-7b)
+- [mosaicml/mpt-7b-chat](https://huggingface.co/mosaicml/mpt-7b-chat)
+  - example: `python3 -m fastchat.serve.cli --model-path mosaicml/mpt-7b-chat`
+- [Neutralzz/BiLLa-7B-SFT](https://huggingface.co/Neutralzz/BiLLa-7B-SFT)
+- [nomic-ai/gpt4all-13b-snoozy](https://huggingface.co/nomic-ai/gpt4all-13b-snoozy)
+- [NousResearch/Nous-Hermes-13b](https://huggingface.co/NousResearch/Nous-Hermes-13b)
+- [openaccess-ai-collective/manticore-13b-chat-pyg](https://huggingface.co/openaccess-ai-collective/manticore-13b-chat-pyg)
+- [OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5](https://huggingface.co/OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5)
+- [openchat/openchat_3.5](https://huggingface.co/openchat/openchat_3.5)
+- [Open-Orca/Mistral-7B-OpenOrca](https://huggingface.co/Open-Orca/Mistral-7B-OpenOrca)
+- [OpenLemur/lemur-70b-chat-v1](https://huggingface.co/OpenLemur/lemur-70b-chat-v1)
+- [Phind/Phind-CodeLlama-34B-v2](https://huggingface.co/Phind/Phind-CodeLlama-34B-v2)
+- [project-baize/baize-v2-7b](https://huggingface.co/project-baize/baize-v2-7b)
+- [Qwen/Qwen-7B-Chat](https://huggingface.co/Qwen/Qwen-7B-Chat)
+- [rishiraj/CatPPT](https://huggingface.co/rishiraj/CatPPT)
+- [Salesforce/codet5p-6b](https://huggingface.co/Salesforce/codet5p-6b)
+- [StabilityAI/stablelm-tuned-alpha-7b](https://huggingface.co/stabilityai/stablelm-tuned-alpha-7b)
+- [tenyx/TenyxChat-7B-v1](https://huggingface.co/tenyx/TenyxChat-7B-v1)
+- [TinyLlama/TinyLlama-1.1B-Chat-v1.0](https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0)
+- [THUDM/chatglm-6b](https://huggingface.co/THUDM/chatglm-6b)
+- [THUDM/chatglm2-6b](https://huggingface.co/THUDM/chatglm2-6b)
+- [tiiuae/falcon-40b](https://huggingface.co/tiiuae/falcon-40b)
+- [tiiuae/falcon-180B-chat](https://huggingface.co/tiiuae/falcon-180B-chat)
+- [timdettmers/guanaco-33b-merged](https://huggingface.co/timdettmers/guanaco-33b-merged)
+- [togethercomputer/RedPajama-INCITE-7B-Chat](https://huggingface.co/togethercomputer/RedPajama-INCITE-7B-Chat)
+- [VMware/open-llama-7b-v2-open-instruct](https://huggingface.co/VMware/open-llama-7b-v2-open-instruct)
+- [WizardLM/WizardLM-13B-V1.0](https://huggingface.co/WizardLM/WizardLM-13B-V1.0)
+- [WizardLM/WizardCoder-15B-V1.0](https://huggingface.co/WizardLM/WizardCoder-15B-V1.0)
+- [Xwin-LM/Xwin-LM-7B-V0.1](https://huggingface.co/Xwin-LM/Xwin-LM-70B-V0.1)
+- Any [EleutherAI](https://huggingface.co/EleutherAI) pythia model such as [pythia-6.9b](https://huggingface.co/EleutherAI/pythia-6.9b)
+- Any [Peft](https://github.com/huggingface/peft) adapter trained on top of a
+  model above.  To activate, must have `peft` in the model path.  Note: If
+  loading multiple peft models, you can have them share the base model weights by
+  setting the environment variable `PEFT_SHARE_BASE_WEIGHTS=true` in any model
+  worker.
+
+
+## API-Based Models
+To support an API-based model, consider learning from the existing OpenAI example.
+If the model is compatible with OpenAI APIs, then a configuration file is all that's needed without any additional code.
+For custom protocols, implementation of a streaming generator in [fastchat/serve/api_provider.py](https://github.com/lm-sys/FastChat/blob/main/fastchat/serve/api_provider.py) is required, following the provided examples. Currently, FastChat is compatible with OpenAI, Anthropic, Google Vertex AI, Mistral, and Nvidia NGC.
+
+### Steps to Launch a WebUI with an API Model
+1. Specify the endpoint information in a JSON configuration file. For instance, create a file named `api_endpoints.json`:
+```json
+{
+  "gpt-3.5-turbo": {
+    "model_name": "gpt-3.5-turbo",
+    "api_type": "openai",
+    "api_base": "https://api.openai.com/v1",
+    "api_key": "sk-******",
+    "anony_only": false
+  }
+}
+```
+  - "api_type" can be one of the following: openai, anthropic, gemini, or mistral. For custom APIs, add a new type and implement it accordingly.
+  - "anony_only" indicates whether to display this model in anonymous mode only.
+
+2. Launch the Gradio web server with the argument `--register api_endpoints.json`:
+```
+python3 -m fastchat.serve.gradio_web_server --controller "" --share --register api_endpoints.json
+```
+
+Now, you can open a browser and interact with the model.

docs/openai_api.md

@@ -0,0 +1,152 @@
+# OpenAI-Compatible RESTful APIs
+
+FastChat provides OpenAI-compatible APIs for its supported models, so you can use FastChat as a local drop-in replacement for OpenAI APIs.
+The FastChat server is compatible with both [openai-python](https://github.com/openai/openai-python) library and cURL commands.
+
+The following OpenAI APIs are supported:
+- Chat Completions. (Reference: https://platform.openai.com/docs/api-reference/chat)
+- Completions. (Reference: https://platform.openai.com/docs/api-reference/completions)
+- Embeddings. (Reference: https://platform.openai.com/docs/api-reference/embeddings)
+
+The REST API can be seamlessly operated from Google Colab, as demonstrated in the [FastChat_API_GoogleColab.ipynb](https://github.com/lm-sys/FastChat/blob/main/playground/FastChat_API_GoogleColab.ipynb) notebook, available in our repository. This notebook provides a practical example of how to utilize the API effectively within the Google Colab environment.
+
+## RESTful API Server
+First, launch the controller
+
+```bash
+python3 -m fastchat.serve.controller
+```
+
+Then, launch the model worker(s)
+
+```bash
+python3 -m fastchat.serve.model_worker --model-path lmsys/vicuna-7b-v1.5
+```
+
+Finally, launch the RESTful API server
+
+```bash
+python3 -m fastchat.serve.openai_api_server --host localhost --port 8000
+```
+
+Now, let us test the API server.
+
+### OpenAI Official SDK
+The goal of `openai_api_server.py` is to implement a fully OpenAI-compatible API server, so the models can be used directly with [openai-python](https://github.com/openai/openai-python) library.
+
+First, install OpenAI python package >= 1.0:
+```bash
+pip install --upgrade openai
+```
+
+Then, interact with the Vicuna model:
+```python
+import openai
+
+openai.api_key = "EMPTY"
+openai.base_url = "http://localhost:8000/v1/"
+
+model = "vicuna-7b-v1.5"
+prompt = "Once upon a time"
+
+# create a completion
+completion = openai.completions.create(model=model, prompt=prompt, max_tokens=64)
+# print the completion
+print(prompt + completion.choices[0].text)
+
+# create a chat completion
+completion = openai.chat.completions.create(
+  model=model,
+  messages=[{"role": "user", "content": "Hello! What is your name?"}]
+)
+# print the completion
+print(completion.choices[0].message.content)
+```
+
+Streaming is also supported. See [test_openai_api.py](../tests/test_openai_api.py).  If your api server is behind a proxy you'll need to turn off buffering, you can do so in Nginx by setting `proxy_buffering off;` in the location block for the proxy.
+
+### cURL
+cURL is another good tool for observing the output of the api.
+
+List Models:
+```bash
+curl http://localhost:8000/v1/models
+```
+
+Chat Completions:
+```bash
+curl http://localhost:8000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "vicuna-7b-v1.5",
+    "messages": [{"role": "user", "content": "Hello! What is your name?"}]
+  }'
+```
+
+Text Completions:
+```bash
+curl http://localhost:8000/v1/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "vicuna-7b-v1.5",
+    "prompt": "Once upon a time",
+    "max_tokens": 41,
+    "temperature": 0.5
+  }'
+```
+
+Embeddings:
+```bash
+curl http://localhost:8000/v1/embeddings \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "vicuna-7b-v1.5",
+    "input": "Hello world!"
+  }'
+```
+
+### Running multiple 
+
+If you want to run multiple models on the same machine and in the same process,
+you can replace the `model_worker` step above with a multi model variant:
+
+```bash
+python3 -m fastchat.serve.multi_model_worker \
+    --model-path lmsys/vicuna-7b-v1.5 \
+    --model-names vicuna-7b-v1.5 \
+    --model-path lmsys/longchat-7b-16k \
+    --model-names longchat-7b-16k
+```
+
+This loads both models into the same accelerator and in the same process.  This
+works best when using a Peft model that triggers the `PeftModelAdapter`.
+
+TODO: Base model weight optimization will be fixed once [this
+Peft](https://github.com/huggingface/peft/issues/430) issue is resolved.
+
+## LangChain Support
+This OpenAI-compatible API server supports LangChain. See [LangChain Integration](langchain_integration.md) for details.
+
+## Adjusting Environment Variables
+
+### Timeout
+By default, a timeout error will occur if a model worker does not response within 100 seconds. If your model/hardware is slower, you can change this timeout through an environment variable: 
+
+```bash
+export FASTCHAT_WORKER_API_TIMEOUT=<larger timeout in seconds>
+```
+
+### Batch size
+If you meet the following OOM error while creating embeddings. You can use a smaller batch size by setting
+
+```bash
+export FASTCHAT_WORKER_API_EMBEDDING_BATCH_SIZE=1
+```
+
+## Todos
+Some features to be implemented:
+
+- [ ] Support more parameters like `logprobs`, `logit_bias`, `user`, `presence_penalty` and `frequency_penalty`
+- [ ] Model details (permissions, owner and create time)
+- [ ] Edits API
+- [ ] Rate Limitation Settings

docs/server_arch.md

@@ -0,0 +1,2 @@
+# FastChat Server Architecture
+![server arch](../assets/server_arch.png)

docs/third_party_ui.md

@@ -0,0 +1,24 @@
+# Third Party UI
+If you want to host it on your own UI or third party UI, you can launch the [OpenAI compatible server](openai_api.md) and host with a tunnelling service such as Tunnelmole or ngrok, and then enter the credentials appropriately.
+
+You can find suitable UIs from third party repos:
+- [WongSaang's ChatGPT UI](https://github.com/WongSaang/chatgpt-ui)
+- [McKayWrigley's Chatbot UI](https://github.com/mckaywrigley/chatbot-ui)
+
+- Please note that some third-party providers only offer the standard `gpt-3.5-turbo`, `gpt-4`, etc., so you will have to add your own custom model inside the code. [Here is an example of how to create a UI with any custom model name](https://github.com/ztjhz/BetterChatGPT/pull/461).
+
+##### Using Tunnelmole
+Tunnelmole is an open source tunnelling tool. You can find its source code on [Github](https://github.com/robbie-cahill/tunnelmole-client). Here's how you can use Tunnelmole:
+1. Install Tunnelmole with `curl -O https://install.tunnelmole.com/9Wtxu/install && sudo bash install`. (On Windows, download [tmole.exe](https://tunnelmole.com/downloads/tmole.exe)). Head over to the [README](https://github.com/robbie-cahill/tunnelmole-client) for other methods such as `npm` or building from source.
+2. Run `tmole 7860` (replace `7860` with your listening port if it is different from 7860). The output will display two URLs: one HTTP and one HTTPS. It's best to use the HTTPS URL for better privacy and security.
+```
+➜  ~ tmole 7860
+http://bvdo5f-ip-49-183-170-144.tunnelmole.net is forwarding to localhost:7860
+https://bvdo5f-ip-49-183-170-144.tunnelmole.net is forwarding to localhost:7860
+```
+
+##### Using ngrok
+ngrok is a popular closed source tunnelling tool. First download and install it from [ngrok.com](https://ngrok.com/downloads). Here's how to use it to expose port 7860.
+```
+ngrok http 7860
+```

docs/training.md

@@ -0,0 +1,118 @@
+### Fine-tuning FastChat-T5
+You can use the following command to train FastChat-T5 with 4 x A100 (40GB).
+```bash
+torchrun --nproc_per_node=4 --master_port=9778 fastchat/train/train_flant5.py \
+    --model_name_or_path google/flan-t5-xl \
+    --data_path ./data/dummy_conversation.json \
+    --bf16 True \
+    --output_dir ./checkpoints_flant5_3b \
+    --num_train_epochs 3 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --gradient_accumulation_steps 4 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 300 \
+    --save_total_limit 1 \
+    --learning_rate 2e-5 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --fsdp "full_shard auto_wrap" \
+    --fsdp_transformer_layer_cls_to_wrap T5Block \
+    --tf32 True \
+    --model_max_length 2048 \
+    --preprocessed_path ./preprocessed_data/processed.json \
+    --gradient_checkpointing True 
+```
+
+After training, please use our post-processing [function](https://github.com/lm-sys/FastChat/blob/55051ad0f23fef5eeecbda14a2e3e128ffcb2a98/fastchat/utils.py#L166-L185) to update the saved model weight. Additional discussions can be found [here](https://github.com/lm-sys/FastChat/issues/643).
+
+### Fine-tuning using (Q)LoRA
+You can use the following command to train Vicuna-7B using QLoRA using ZeRO2. Note that ZeRO3 is not currently supported with QLoRA but ZeRO3 does support LoRA, which has a reference configuraiton under playground/deepspeed_config_s3.json. To use QLoRA, you must have bitsandbytes>=0.39.0 and transformers>=4.30.0 installed.
+```bash
+deepspeed fastchat/train/train_lora.py \
+    --model_name_or_path ~/model_weights/llama-7b \
+    --lora_r 8 \
+    --lora_alpha 16 \
+    --lora_dropout 0.05 \
+    --data_path ./data/dummy_conversation.json \
+    --bf16 True \
+    --output_dir ./checkpoints \
+    --num_train_epochs 3 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --gradient_accumulation_steps 1 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 1200 \
+    --save_total_limit 100 \
+    --learning_rate 2e-5 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 True \
+    --model_max_length 2048 \
+    --q_lora True \
+    --deepspeed playground/deepspeed_config_s2.json \
+```
+
+For T5-XL or XXL
+
+```bash
+deepspeed fastchat/train/train_lora_t5.py \
+        --model_name_or_path google/flan-t5-xl    \
+        --data_path ./data/dummy_conversation.json \
+        --bf16 True \
+        --output_dir ./checkpoints_flant5_3b \
+        --num_train_epochs 3 \
+        --per_device_train_batch_size 1 \
+        --per_device_eval_batch_size 1  \
+        --gradient_accumulation_steps 4  \
+        --evaluation_strategy "no"  \
+        --save_strategy "steps"  \
+        --save_steps 300 \
+        --save_total_limit 1 \
+        --learning_rate 2e-5 \
+        --weight_decay 0.     \
+        --warmup_ratio 0.03    \
+        --lr_scheduler_type "cosine"   \
+        --logging_steps 1 \
+        --model_max_length 2048    \
+        --preprocessed_path ./preprocessed_data/processed.json \
+        --gradient_checkpointing True \
+        --q_lora True     \
+        --deepspeed playground/deepspeed_config_s2.json
+        
+```
+
+### Fine-tuning Vicuna-7B with Local NPUs
+
+You can use the following command to train Vicuna-7B with 8 x NPUs. Use `--nproc_per_node` to specify the number of NPUs.
+```bash
+torchrun --nproc_per_node=8 --master_port=20001 fastchat/train/train.py \
+    --model_name_or_path ~/vicuna-7b-v1.5-16k  \
+    --data_path data/dummy_conversation.json \
+    --fp16 True \
+    --output_dir output_vicuna \
+    --num_train_epochs 3 \
+    --per_device_train_batch_size 8 \
+    --per_device_eval_batch_size 1 \
+    --gradient_accumulation_steps 1 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 1200 \
+    --save_total_limit 10 \
+    --learning_rate 2e-5 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --fsdp "full_shard auto_wrap" \
+    --fsdp_transformer_layer_cls_to_wrap 'LlamaDecoderLayer' \
+    --model_max_length 2048 \
+    --gradient_checkpointing True \
+    --lazy_preprocess True
+```

docs/vicuna_weights_version.md

@@ -0,0 +1,97 @@
+## Vicuna Weights
+
+| Weights version | Link | FastChat version compatibility | Base Model | Release Date | Fine-tuning Data |
+| ---- | ---- | ---- | ---- | ---- | ---- |
+| v1.5 | [7B](https://huggingface.co/lmsys/vicuna-7b-v1.5), [7B-16k](https://huggingface.co/lmsys/vicuna-7b-v1.5-16k), [13B](https://huggingface.co/lmsys/vicuna-13b-v1.5), [13B-16k](https://huggingface.co/lmsys/vicuna-13b-v1.5-16k) | `>=0.2.21` | Llama 2 | Aug. 1, 2023 | 370M tokens |
+| v1.3 | [7B](https://huggingface.co/lmsys/vicuna-7b-v1.3), [13B](https://huggingface.co/lmsys/vicuna-13b-v1.3), [33B](//huggingface.co/lmsys/vicuna-33b-v1.3) | `>=0.2.1` | Llama 1 | Jun. 22, 2023 | 370M tokens |
+| v1.1 | [7B](https://huggingface.co/lmsys/vicuna-7b-v1.1), [13B](https://huggingface.co/lmsys/vicuna-13b-v1.1) | `>=0.2.1` | Llama 1 | Apr. 12, 2023 | - |
+| v0 | [7B-delta](https://huggingface.co/lmsys/vicuna-7b-delta-v0), [13B-delta](https://huggingface.co/lmsys/vicuna-13b-delta-v0) | `<=0.1.10` | Llama 1 | Mar. 30, 2023 | - |
+
+### Updates
+- Major updates of weights v1.5
+  - Use Llama2 as the base model.
+  - Provide 16K context length versions using linear RoPE scaling.
+
+- Major updates of weights v1.3
+  - Train with twice the amount of ShareGPT data compared to previous versions.
+  - Provide merged weights directly instead of delta weights.
+
+- Major updates of weights v1.1
+  - Refactor the tokenization and separator. In Vicuna v1.1, the separator has been changed from `###` to the EOS token `</s>`. This change makes it easier to determine the generation stop criteria and enables better compatibility with other libraries.
+  - Fix the supervised fine-tuning loss computation for better model quality.
+
+## Prompt Template
+
+### Example prompt (weights v1.1, v1.3, v1.5)
+```
+A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.
+
+USER: Hello!
+ASSISTANT: Hello!</s>
+USER: How are you?
+ASSISTANT: I am good.</s>
+```
+
+See a full prompt template [here](https://github.com/lm-sys/FastChat/blob/d578599c69d060e6d40943f1b5b72af98956092a/fastchat/conversation.py#L286-L299) and example output [here](https://github.com/lm-sys/FastChat/blob/d578599c69d060e6d40943f1b5b72af98956092a/fastchat/conversation.py#L748-L753).
+
+### Example prompt (weights v0)
+```
+A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.
+
+### Human: Hello!
+### Assistant: Hello!
+### Human: How are you?
+### Assistant: I am good.
+```
+
+See the full prompt template [here](https://github.com/lm-sys/FastChat/blob/d578599c69d060e6d40943f1b5b72af98956092a/fastchat/conversation.py#L238-L269).
+
+## How to Apply Delta Weights (Only Needed for Weights v0)
+
+We release [Vicuna](https://lmsys.org/blog/2023-03-30-vicuna/) weights v0 as delta weights to comply with the LLaMA model license.
+You can add our delta to the original LLaMA weights to obtain the Vicuna weights. Instructions:
+
+1. Get the original LLaMA weights in the Hugging Face format by following the instructions [here](https://huggingface.co/docs/transformers/main/model_doc/llama).
+2. Use the following scripts to get Vicuna weights by applying our delta. They will automatically download delta weights from our Hugging Face [account](https://huggingface.co/lmsys).
+
+**NOTE**:
+Weights v1.1 are only compatible with ```transformers>=4.28.0``` and ``fschat >= 0.2.0``.
+Please update your local packages accordingly. If you follow the above commands to do a fresh install, then you should get all the correct versions.
+
+#### Vicuna-7B
+This conversion command needs around 30 GB of CPU RAM.
+See the "Low CPU Memory Conversion" section below if you do not have enough memory.
+Replace `/path/to/*` with the real paths.
+```bash
+python3 -m fastchat.model.apply_delta \
+    --base-model-path /path/to/llama-7b \
+    --target-model-path /path/to/output/vicuna-7b \
+    --delta-path lmsys/vicuna-7b-delta-v1.1
+```
+
+#### Vicuna-13B
+This conversion command needs around 60 GB of CPU RAM.
+See the "Low CPU Memory Conversion" section below if you do not have enough memory.
+Replace `/path/to/*` with the real paths.
+```bash
+python3 -m fastchat.model.apply_delta \
+    --base-model-path /path/to/llama-13b \
+    --target-model-path /path/to/output/vicuna-13b \
+    --delta-path lmsys/vicuna-13b-delta-v1.1
+```
+
+#### Low CPU Memory Conversion
+You can try these methods to reduce the CPU RAM requirement of weight conversion.
+1. Append `--low-cpu-mem` to the commands above, which will split large weight files into smaller ones and use the disk as temporary storage. This can keep the peak memory at less than 16GB.
+2. Create a large swap file and rely on the operating system to automatically utilize the disk as virtual memory.
+
+## FAQ
+
+### Tokenizer issues
+There are some frequently asked tokenizer issues (https://github.com/lm-sys/FastChat/issues/408).
+Some of them are not only related to FastChat or Vicuna weights but are also related to how you convert the base llama model.
+
+We suggest that you use `transformers>=4.28.0` and redo the weight conversion for the base llama model.
+After applying the delta, you should have a file named `special_tokens_map.json` in your converted weight folder for either v0 or v1.1.
+The contents of this file should be the same as this file: https://huggingface.co/lmsys/vicuna-13b-delta-v0/blob/main/special_tokens_map.json.
+If the file is not present, please copy the `special_tokens_map.json` and `tokenizer_config.json` files from https://huggingface.co/lmsys/vicuna-13b-delta-v0/tree/main to your converted weight folder. This works for both v0 and v1.1.

docs/vllm_integration.md

@@ -0,0 +1,25 @@
+# vLLM Integration
+You can use [vLLM](https://vllm.ai/) as an optimized worker implementation in FastChat.
+It offers advanced continuous batching and a much higher (~10x) throughput.
+See the supported models [here](https://vllm.readthedocs.io/en/latest/models/supported_models.html).
+
+## Instructions
+1. Install vLLM.
+    ```
+    pip install vllm
+    ```
+
+2. When you launch a model worker, replace the normal worker (`fastchat.serve.model_worker`) with the vLLM worker (`fastchat.serve.vllm_worker`). All other commands such as controller, gradio web server, and OpenAI API server are kept the same.
+   ```
+   python3 -m fastchat.serve.vllm_worker --model-path lmsys/vicuna-7b-v1.5
+   ```
+
+   If you see tokenizer errors, try
+   ```
+   python3 -m fastchat.serve.vllm_worker --model-path lmsys/vicuna-7b-v1.5 --tokenizer hf-internal-testing/llama-tokenizer
+   ```
+
+   If you use an AWQ quantized model, try
+   '''
+   python3 -m fastchat.serve.vllm_worker --model-path TheBloke/vicuna-7B-v1.5-AWQ --quantization awq
+   '''

docs/xFasterTransformer.md

@@ -0,0 +1,90 @@
+# xFasterTransformer Inference Framework
+
+Integrated [xFasterTransformer](https://github.com/intel/xFasterTransformer) customized framework into Fastchat to provide **Faster** inference speed on Intel CPU.
+
+## Install xFasterTransformer
+
+Setup environment (please refer to [this link](https://github.com/intel/xFasterTransformer#installation) for more details):
+
+```bash
+pip install xfastertransformer
+```
+
+## Prepare models
+
+Prepare Model (please refer to [this link](https://github.com/intel/xFasterTransformer#prepare-model) for more details):
+```bash
+python ./tools/chatglm_convert.py -i ${HF_DATASET_DIR} -o  ${OUTPUT_DIR}
+```
+
+## Parameters of xFasterTransformer
+--enable-xft to enable xfastertransformer in Fastchat
+--xft-max-seq-len to set the max token length the model can process. max token length include input token length.
+--xft-dtype to set datatype used in xFasterTransformer for computation. xFasterTransformer can support fp32, fp16, int8, bf16 and hybrid data types like : bf16_fp16, bf16_int8. For datatype details please refer to [this link](https://github.com/intel/xFasterTransformer/wiki/Data-Type-Support-Platform)
+    
+
+Chat with the CLI:
+```bash
+#run inference on all CPUs and using float16
+python3 -m fastchat.serve.cli \
+    --model-path /path/to/models \
+    --enable-xft \
+    --xft-dtype fp16
+```
+or with numactl on multi-socket server for better performance
+```bash
+#run inference on numanode 0 and with data type bf16_fp16 (first token uses bfloat16, and rest tokens use float16)
+numactl -N 0  --localalloc \
+python3 -m fastchat.serve.cli \
+    --model-path /path/to/models/chatglm2_6b_cpu/ \
+    --enable-xft \
+    --xft-dtype bf16_fp16
+```
+or using MPI to run inference on 2 sockets for better performance
+```bash
+#run inference on numanode 0 and 1 and with data type bf16_fp16 (first token uses bfloat16, and rest tokens use float16)
+OMP_NUM_THREADS=$CORE_NUM_PER_SOCKET LD_PRELOAD=libiomp5.so mpirun \
+-n 1 numactl -N 0  --localalloc \
+python -m fastchat.serve.cli \ 
+    --model-path /path/to/models/chatglm2_6b_cpu/ \
+    --enable-xft \
+    --xft-dtype bf16_fp16 : \
+-n 1 numactl -N 1  --localalloc \
+python -m fastchat.serve.cli \
+    --model-path /path/to/models/chatglm2_6b_cpu/ \
+    --enable-xft \
+    --xft-dtype bf16_fp16
+```
+
+
+Start model worker:
+```bash
+# Load model with default configuration (max sequence length 4096, no GPU split setting).
+python3 -m fastchat.serve.model_worker \
+    --model-path /path/to/models \
+    --enable-xft \
+    --xft-dtype bf16_fp16 
+```
+or with numactl on multi-socket server for better performance
+```bash
+#run inference on numanode 0 and with data type bf16_fp16 (first token uses bfloat16, and rest tokens use float16)
+numactl -N 0  --localalloc python3 -m fastchat.serve.model_worker \
+    --model-path /path/to/models \
+    --enable-xft \
+    --xft-dtype bf16_fp16 
+```
+or using MPI to run inference on 2 sockets for better performance
+```bash
+#run inference on numanode 0 and 1 and with data type bf16_fp16 (first token uses bfloat16, and rest tokens use float16)
+OMP_NUM_THREADS=$CORE_NUM_PER_SOCKET LD_PRELOAD=libiomp5.so mpirun \
+-n 1 numactl -N 0  --localalloc  python -m fastchat.serve.model_worker \
+    --model-path /path/to/models \
+    --enable-xft \
+    --xft-dtype bf16_fp16 : \
+-n 1 numactl -N 1  --localalloc  python -m fastchat.serve.model_worker \
+    --model-path /path/to/models \
+    --enable-xft \
+    --xft-dtype bf16_fp16 
+```
+
+For more details, please refer to [this link](https://github.com/intel/xFasterTransformer#how-to-run) 

fastchat/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.35"