From 56465437fe17d8ef5ad7fbc36e57327849909e63 Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Fri, 20 Jul 2018 09:53:38 -0300 Subject: [PATCH 01/23] Init benchmark tool --- .gitignore | 1 + benchmark/Pipfile | 13 ++++ benchmark/Pipfile.lock | 66 +++++++++++++++++++ benchmark/nlu_benchmark/__init__.py | 0 benchmark/nlu_benchmark/cli/__init__.py | 0 benchmark/nlu_benchmark/cli/__main__.py | 16 +++++ benchmark/nlu_benchmark/cli/tools/__init__.py | 0 benchmark/nlu_benchmark/cli/tools/bothub.py | 16 +++++ benchmark/nlu_benchmark/services/__init__.py | 0 benchmark/nlu_benchmark/services/bothub.py | 31 +++++++++ benchmark/nlu_benchmark/services/service.py | 2 + 11 files changed, 145 insertions(+) create mode 100644 .gitignore create mode 100644 benchmark/Pipfile create mode 100644 benchmark/Pipfile.lock create mode 100644 benchmark/nlu_benchmark/__init__.py create mode 100644 benchmark/nlu_benchmark/cli/__init__.py create mode 100644 benchmark/nlu_benchmark/cli/__main__.py create mode 100644 benchmark/nlu_benchmark/cli/tools/__init__.py create mode 100644 benchmark/nlu_benchmark/cli/tools/bothub.py create mode 100644 benchmark/nlu_benchmark/services/__init__.py create mode 100644 benchmark/nlu_benchmark/services/bothub.py create mode 100644 benchmark/nlu_benchmark/services/service.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dbe9c82 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.vscode/ \ No newline at end of file diff --git a/benchmark/Pipfile b/benchmark/Pipfile new file mode 100644 index 0000000..aaa0dc8 --- /dev/null +++ b/benchmark/Pipfile @@ -0,0 +1,13 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +plac = "*" +requests = "*" + +[dev-packages] + +[requires] +python_version = "3.6" diff --git a/benchmark/Pipfile.lock b/benchmark/Pipfile.lock new file mode 100644 index 0000000..5d6e25e --- /dev/null +++ b/benchmark/Pipfile.lock @@ -0,0 +1,66 @@ +{ + "_meta": { + "hash": { + "sha256": "7e83e40d821e8d569bf551375c17fe86300ff929c0808101ea4f27b8e5380acb" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.6" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "certifi": { + "hashes": [ + "sha256:13e698f54293db9f89122b0581843a782ad0934a4fe0172d2a980ba77fc61bb7", + "sha256:9fa520c1bacfb634fa7af20a76bcbd3d5fb390481724c597da32c719a7dca4b0" + ], + "version": "==2018.4.16" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "idna": { + "hashes": [ + "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", + "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" + ], + "version": "==2.7" + }, + "plac": { + "hashes": [ + "sha256:854693ad90367e8267112ffbb8955f57d6fdeac3191791dc9ffce80f87fd2370", + "sha256:ba3f719a018175f0a15a6b04e6cc79c25fd563d348aacd320c3644d2a9baf89b" + ], + "index": "pypi", + "version": "==0.9.6" + }, + "requests": { + "hashes": [ + "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", + "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a" + ], + "index": "pypi", + "version": "==2.19.1" + }, + "urllib3": { + "hashes": [ + "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", + "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" + ], + "markers": "python_version != '3.0.*' and python_version < '4' and python_version != '3.2.*' and python_version != '3.1.*' and python_version != '3.3.*' and python_version >= '2.6'", + "version": "==1.23" + } + }, + "develop": {} +} diff --git a/benchmark/nlu_benchmark/__init__.py b/benchmark/nlu_benchmark/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmark/nlu_benchmark/cli/__init__.py b/benchmark/nlu_benchmark/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmark/nlu_benchmark/cli/__main__.py b/benchmark/nlu_benchmark/cli/__main__.py new file mode 100644 index 0000000..8fe82ae --- /dev/null +++ b/benchmark/nlu_benchmark/cli/__main__.py @@ -0,0 +1,16 @@ +import sys +import plac + +from .tools.bothub import get_token as bothub_get_token + + +commands = { + 'bothub_get_token': bothub_get_token, +} + +@plac.annotations( + command=plac.Annotation(choices=commands.keys())) +def main(command, *args): + plac.call(commands.get(command), args) + +plac.call(main, sys.argv[1:]) \ No newline at end of file diff --git a/benchmark/nlu_benchmark/cli/tools/__init__.py b/benchmark/nlu_benchmark/cli/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmark/nlu_benchmark/cli/tools/bothub.py b/benchmark/nlu_benchmark/cli/tools/bothub.py new file mode 100644 index 0000000..a08f783 --- /dev/null +++ b/benchmark/nlu_benchmark/cli/tools/bothub.py @@ -0,0 +1,16 @@ +import plac + +from getpass import getpass + +from nlu_benchmark.services.bothub import Bothub + + +@plac.annotations( + username=plac.Annotation(kind='option'), + password=plac.Annotation(kind='option')) +def get_token(username=None, password=None): + if not username: + username = input('username: ') + if not password: + password = getpass('password: ') + print(Bothub.login(username, password)) diff --git a/benchmark/nlu_benchmark/services/__init__.py b/benchmark/nlu_benchmark/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmark/nlu_benchmark/services/bothub.py b/benchmark/nlu_benchmark/services/bothub.py new file mode 100644 index 0000000..9c57b7f --- /dev/null +++ b/benchmark/nlu_benchmark/services/bothub.py @@ -0,0 +1,31 @@ +from requests import Request, Session + +from .service import Service + + +class Bothub(Service): + @classmethod + def api_request(cls, path, data=None): + session = Session() + request = Request( + 'POST' if data else 'GET', + f'https://bothub.it/api/{path}/', + data=data, + headers={}) + prepped = request.prepare() + return session.send(prepped) + + + @classmethod + def login(cls, username, password): + response = cls.api_request( + 'login', + {'username': username, 'password': password}) + try: + response.raise_for_status() + return response.json().get('token') + except Exception as e: + raise Exception(' / '.join([ + f"{field}: {'; '.join(errors)}" for field, errors in + response.json().items() + ])) \ No newline at end of file diff --git a/benchmark/nlu_benchmark/services/service.py b/benchmark/nlu_benchmark/services/service.py new file mode 100644 index 0000000..938f7b7 --- /dev/null +++ b/benchmark/nlu_benchmark/services/service.py @@ -0,0 +1,2 @@ +class Service: + pass \ No newline at end of file From bf57ee3e8c6dff49237cffa7abeef436c28175c9 Mon Sep 17 00:00:00 2001 From: periclesJ Date: Wed, 25 Jul 2018 14:52:03 -0300 Subject: [PATCH 02/23] Added first version of the automated test --- benchmark/nlu_benchmark/services/LICENSE | 674 ++++++++++++++++++ benchmark/nlu_benchmark/services/Pipfile | 13 + benchmark/nlu_benchmark/services/Pipfile.lock | 135 ++++ .../nlu_benchmark/services/app/__init__.py | 0 .../nlu_benchmark/services/app/__main__.py | 82 +++ .../nlu_benchmark/services/app/bothub.py | 34 + benchmark/nlu_benchmark/services/app/utils.py | 11 + benchmark/nlu_benchmark/services/app/wit.py | 40 ++ 8 files changed, 989 insertions(+) create mode 100644 benchmark/nlu_benchmark/services/LICENSE create mode 100644 benchmark/nlu_benchmark/services/Pipfile create mode 100644 benchmark/nlu_benchmark/services/Pipfile.lock create mode 100644 benchmark/nlu_benchmark/services/app/__init__.py create mode 100644 benchmark/nlu_benchmark/services/app/__main__.py create mode 100644 benchmark/nlu_benchmark/services/app/bothub.py create mode 100644 benchmark/nlu_benchmark/services/app/utils.py create mode 100644 benchmark/nlu_benchmark/services/app/wit.py diff --git a/benchmark/nlu_benchmark/services/LICENSE b/benchmark/nlu_benchmark/services/LICENSE new file mode 100644 index 0000000..94a9ed0 --- /dev/null +++ b/benchmark/nlu_benchmark/services/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/benchmark/nlu_benchmark/services/Pipfile b/benchmark/nlu_benchmark/services/Pipfile new file mode 100644 index 0000000..9bd92f2 --- /dev/null +++ b/benchmark/nlu_benchmark/services/Pipfile @@ -0,0 +1,13 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +requests = "*" + +[dev-packages] +pylint = "*" + +[requires] +python_version = "3.6" diff --git a/benchmark/nlu_benchmark/services/Pipfile.lock b/benchmark/nlu_benchmark/services/Pipfile.lock new file mode 100644 index 0000000..2297a09 --- /dev/null +++ b/benchmark/nlu_benchmark/services/Pipfile.lock @@ -0,0 +1,135 @@ +{ + "_meta": { + "hash": { + "sha256": "5385c98d02bfcf11fb6a74ae72ea941086c6324557da061a89eea4194546523e" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.6" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "certifi": { + "hashes": [ + "sha256:13e698f54293db9f89122b0581843a782ad0934a4fe0172d2a980ba77fc61bb7", + "sha256:9fa520c1bacfb634fa7af20a76bcbd3d5fb390481724c597da32c719a7dca4b0" + ], + "version": "==2018.4.16" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "idna": { + "hashes": [ + "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", + "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" + ], + "version": "==2.7" + }, + "requests": { + "hashes": [ + "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", + "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a" + ], + "index": "pypi", + "version": "==2.19.1" + }, + "urllib3": { + "hashes": [ + "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", + "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" + ], + "version": "==1.23" + } + }, + "develop": { + "astroid": { + "hashes": [ + "sha256:0ef2bf9f07c3150929b25e8e61b5198c27b0dca195e156f0e4d5bdd89185ca1a", + "sha256:fc9b582dba0366e63540982c3944a9230cbc6f303641c51483fa547dcc22393a" + ], + "version": "==1.6.5" + }, + "isort": { + "hashes": [ + "sha256:1153601da39a25b14ddc54955dbbacbb6b2d19135386699e2ad58517953b34af", + "sha256:b9c40e9750f3d77e6e4d441d8b0266cf555e7cdabdcff33c4fd06366ca761ef8", + "sha256:ec9ef8f4a9bc6f71eec99e1806bfa2de401650d996c59330782b89a5555c1497" + ], + "version": "==4.3.4" + }, + "lazy-object-proxy": { + "hashes": [ + "sha256:0ce34342b419bd8f018e6666bfef729aec3edf62345a53b537a4dcc115746a33", + "sha256:1b668120716eb7ee21d8a38815e5eb3bb8211117d9a90b0f8e21722c0758cc39", + "sha256:209615b0fe4624d79e50220ce3310ca1a9445fd8e6d3572a896e7f9146bbf019", + "sha256:27bf62cb2b1a2068d443ff7097ee33393f8483b570b475db8ebf7e1cba64f088", + "sha256:27ea6fd1c02dcc78172a82fc37fcc0992a94e4cecf53cb6d73f11749825bd98b", + "sha256:2c1b21b44ac9beb0fc848d3993924147ba45c4ebc24be19825e57aabbe74a99e", + "sha256:2df72ab12046a3496a92476020a1a0abf78b2a7db9ff4dc2036b8dd980203ae6", + "sha256:320ffd3de9699d3892048baee45ebfbbf9388a7d65d832d7e580243ade426d2b", + "sha256:50e3b9a464d5d08cc5227413db0d1c4707b6172e4d4d915c1c70e4de0bbff1f5", + "sha256:5276db7ff62bb7b52f77f1f51ed58850e315154249aceb42e7f4c611f0f847ff", + "sha256:61a6cf00dcb1a7f0c773ed4acc509cb636af2d6337a08f362413c76b2b47a8dd", + "sha256:6ae6c4cb59f199d8827c5a07546b2ab7e85d262acaccaacd49b62f53f7c456f7", + "sha256:7661d401d60d8bf15bb5da39e4dd72f5d764c5aff5a86ef52a042506e3e970ff", + "sha256:7bd527f36a605c914efca5d3d014170b2cb184723e423d26b1fb2fd9108e264d", + "sha256:7cb54db3535c8686ea12e9535eb087d32421184eacc6939ef15ef50f83a5e7e2", + "sha256:7f3a2d740291f7f2c111d86a1c4851b70fb000a6c8883a59660d95ad57b9df35", + "sha256:81304b7d8e9c824d058087dcb89144842c8e0dea6d281c031f59f0acf66963d4", + "sha256:933947e8b4fbe617a51528b09851685138b49d511af0b6c0da2539115d6d4514", + "sha256:94223d7f060301b3a8c09c9b3bc3294b56b2188e7d8179c762a1cda72c979252", + "sha256:ab3ca49afcb47058393b0122428358d2fbe0408cf99f1b58b295cfeb4ed39109", + "sha256:bd6292f565ca46dee4e737ebcc20742e3b5be2b01556dafe169f6c65d088875f", + "sha256:cb924aa3e4a3fb644d0c463cad5bc2572649a6a3f68a7f8e4fbe44aaa6d77e4c", + "sha256:d0fc7a286feac9077ec52a927fc9fe8fe2fabab95426722be4c953c9a8bede92", + "sha256:ddc34786490a6e4ec0a855d401034cbd1242ef186c20d79d2166d6a4bd449577", + "sha256:e34b155e36fa9da7e1b7c738ed7767fc9491a62ec6af70fe9da4a057759edc2d", + "sha256:e5b9e8f6bda48460b7b143c3821b21b452cb3a835e6bbd5dd33aa0c8d3f5137d", + "sha256:e81ebf6c5ee9684be8f2c87563880f93eedd56dd2b6146d8a725b50b7e5adb0f", + "sha256:eb91be369f945f10d3a49f5f9be8b3d0b93a4c2be8f8a5b83b0571b8123e0a7a", + "sha256:f460d1ceb0e4a5dcb2a652db0904224f367c9b3c1470d5a7683c0480e582468b" + ], + "version": "==1.3.1" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, + "pylint": { + "hashes": [ + "sha256:a48070545c12430cfc4e865bf62f5ad367784765681b3db442d8230f0960aa3c", + "sha256:fff220bcb996b4f7e2b0f6812fd81507b72ca4d8c4d05daf2655c333800cb9b3" + ], + "index": "pypi", + "version": "==1.9.2" + }, + "six": { + "hashes": [ + "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", + "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + ], + "version": "==1.11.0" + }, + "wrapt": { + "hashes": [ + "sha256:d4d560d479f2c21e1b5443bbd15fe7ec4b37fe7e53d335d3b9b0a7b1226fe3c6" + ], + "version": "==1.10.11" + } + } +} diff --git a/benchmark/nlu_benchmark/services/app/__init__.py b/benchmark/nlu_benchmark/services/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmark/nlu_benchmark/services/app/__main__.py b/benchmark/nlu_benchmark/services/app/__main__.py new file mode 100644 index 0000000..878400a --- /dev/null +++ b/benchmark/nlu_benchmark/services/app/__main__.py @@ -0,0 +1,82 @@ +import argparse +import traceback + +from app import bothub +from app.bothub import save_on_bothub, analyze, train +from app.utils import load_json_file#,percentage + +parser = argparse.ArgumentParser(description='Train and Test the accuracy from Bothub') + +sub_tasks = parser.add_subparsers(title='Tasks') + + +def fill_bothub(args): + expressions = load_json_file(args.filename) + + for n, expression in enumerate(expressions['data']): + try: + entity = expression['entities'][0] + entity_name = entity['entity'] + + if entity_name == args.intent: + save_on_bothub(args, expression['text'], entity['value']) + except KeyError: + traceback.print_exc() + print('Skipping expression {} due to an error'.format(expression)) + +# Add another argument that stores a description of the test +task_fill_bothub = sub_tasks.add_parser('fill_bothub', help='Insert data from a source to Bothub') +task_fill_bothub.add_argument('--repository', help='repository uuid destination on bothub') +# task_fill_bothub.add_argument('--source', help='path to the source file, e.g. intents.json') +task_fill_bothub.add_argument('--intent', help='name of the entity that represents the intents') +task_fill_bothub.add_argument('--filename', help='name of the file that contains the phrases to be trained') +task_fill_bothub.add_argument('--testfilename', help='name of the file that contains the phrases to be tested') +task_fill_bothub.set_defaults(func=fill_bothub) + + +def predict(args): + phrases_count = 0 + sum_bothub_hits = 0 + expressions = load_json_file(args.testfilename) + for expression in expressions['data']: + phrases_count += 1 + try: + bothub_result = bothub.analyze(expression['value'], args.lang) + if bothub_result['answer']['intent']['name'] == expression[args.intent]: + sum_bothub_hits += 1 + print(expression['value']+' '+expression['intent']+' - OK') #+bothub_result['answer']['intent']['confidence'] + except KeyError: + traceback.print_exc() + print('Skipping expression {} due to an error'.format(expression)) + + print('============================ RESULT ================================') + # bothub_accuracy = percentage(sum_bothub_hits, count) + # bothub_confidence_avg = sum_bothub_confidence/count + + print('Bothub:') + print('Analized phrases: {}'.format(phrases_count)) + print('Correct predictions: {}'.format(sum_bothub_hits)) + # print('Final Accuracy: {}'.format(bothub_accuracy)) + # print('Average Confidence: {}%'.format(bothub_confidence_avg)) + + +task_predict = sub_tasks.add_parser('predict', help='Predict from Bothub and check accuracy') +# task_predict.add_argument('--authorization-bothub', help='authorization token from dataset on Bothub') +# task_predict.add_argument('--source', help='path to the source file, e.g. intents.json') +task_predict.add_argument('--testfilename', help='name of the source file') +task_predict.add_argument('--intent', help='name of the entity that represents the intents') +task_predict.add_argument('--lang', help='language of the bot on Bothub') +task_predict.set_defaults(func=predict) + +def train_repository(args): + train(args.ownernick, args.slug) + + +task_train = sub_tasks.add_parser('train_repository', help='Train all data uploaded to Bothub') +task_train.add_argument('--slug', help='slog fromt he repository') +task_train.add_argument('--ownernick', help='Owner nickname') +task_train.set_defaults(func=train_repository) + +if __name__ == '__main__': + args = parser.parse_args() + args.func(args) diff --git a/benchmark/nlu_benchmark/services/app/bothub.py b/benchmark/nlu_benchmark/services/app/bothub.py new file mode 100644 index 0000000..d71c996 --- /dev/null +++ b/benchmark/nlu_benchmark/services/app/bothub.py @@ -0,0 +1,34 @@ +import requests +import json + +from app.settings import BOTHUB_APP_URL, BOTHUB_USER_TOKEN, BOTHUB_NLP_URL, BOTHUB_NLP_TOKEN + +user_token_header = {'Authorization': BOTHUB_USER_TOKEN} +nlp_authorization_header = {'Authorization': BOTHUB_NLP_TOKEN} + + +def save_on_bothub(args, text, intent): + data = {'repository': args.repository, 'text': text, 'entities': [], 'intent': intent} + response = requests.post('{0}/api/example/new/'.format(BOTHUB_APP_URL), headers=user_token_header, data=data) + print('--> '+text+' {}'.format(response)) + + +def analyze(text, language): + data = {'text': text} + response = requests.post(BOTHUB_NLP_URL, headers=nlp_authorization_header, data=data) + response_json = json.loads(response.content) + return response_json + + +def train(owner_nickname, repository_slug): + params = {'owner__nickname': owner_nickname, 'slug': repository_slug} + response = requests.get('{0}/api/repository/{1}/{2}/train/'.format(BOTHUB_APP_URL,owner_nickname, repository_slug), params=params, headers=user_token_header) + print("Done") + + +def get_intent_data(result): + return result['answer']['intent'] + + +def get_intent_name(result): + return get_intent_data(result)['name'] diff --git a/benchmark/nlu_benchmark/services/app/utils.py b/benchmark/nlu_benchmark/services/app/utils.py new file mode 100644 index 0000000..99e5cbc --- /dev/null +++ b/benchmark/nlu_benchmark/services/app/utils.py @@ -0,0 +1,11 @@ +import json + + +def load_json_file(entity_file): + with open(entity_file) as json_file: + data = json.load(json_file) + return data + + +def percentage(part, whole): + return 100 * float(part) / float(whole) diff --git a/benchmark/nlu_benchmark/services/app/wit.py b/benchmark/nlu_benchmark/services/app/wit.py new file mode 100644 index 0000000..47253d0 --- /dev/null +++ b/benchmark/nlu_benchmark/services/app/wit.py @@ -0,0 +1,40 @@ +import os + +import requests + +from app.utils import load_json_file + + +def get_expressions_data(source): + files = os.listdir(source) + for filename in files: + file_item = os.path.join(source, filename) + + if filename == 'expressions.json': + return load_json_file(file_item) + return None + + +def get_intent_data(result, intent): + entity = result['entities'] + if intent in entity: + return entity[intent][0] + + +def get_intent_from_input(intent, file_item): + entities_files = os.listdir(file_item) + + for entity_filename in entities_files: + entity_file = os.path.join(file_item, entity_filename) + data = load_json_file(entity_file) + + if intent in data['data']['name']: + return data + return None + + +def analyze(text, token): + params = {'q': text} + headers = {'Authorization': token} + response = requests.get('https://api.wit.ai/message', params=params, headers=headers) + return response.json() From ee7fcf9ecdc60dba060287c12dfb90fdea079759 Mon Sep 17 00:00:00 2001 From: periclesJ Date: Tue, 31 Jul 2018 10:38:28 -0300 Subject: [PATCH 03/23] Update that allow automated training and predictions from jsons --- .../nlu_benchmark/services/app/__main__.py | 46 +++++++++++++++---- .../nlu_benchmark/services/app/bothub.py | 38 ++++++++++++--- benchmark/nlu_benchmark/services/app/utils.py | 11 +++-- 3 files changed, 75 insertions(+), 20 deletions(-) diff --git a/benchmark/nlu_benchmark/services/app/__main__.py b/benchmark/nlu_benchmark/services/app/__main__.py index 878400a..770a5ac 100644 --- a/benchmark/nlu_benchmark/services/app/__main__.py +++ b/benchmark/nlu_benchmark/services/app/__main__.py @@ -2,7 +2,7 @@ import traceback from app import bothub -from app.bothub import save_on_bothub, analyze, train +from app.bothub import save_on_bothub, analyze, analyze_text, train, create_new_repository, delete_repository from app.utils import load_json_file#,percentage parser = argparse.ArgumentParser(description='Train and Test the accuracy from Bothub') @@ -12,23 +12,28 @@ def fill_bothub(args): expressions = load_json_file(args.filename) - + repository_data = create_new_repository(args) for n, expression in enumerate(expressions['data']): + entities = [] + if (len(expression['entities']) > 0): + entities = expression['entities'] try: - entity = expression['entities'][0] - entity_name = entity['entity'] - - if entity_name == args.intent: - save_on_bothub(args, expression['text'], entity['value']) + save_on_bothub(repository_data[0], expression['text'], entities, expression['intent']) except KeyError: traceback.print_exc() print('Skipping expression {} due to an error'.format(expression)) + + train_repository(args) + predict(args) + delete_repository(args.ownernick, repository_data[1]) # Add another argument that stores a description of the test task_fill_bothub = sub_tasks.add_parser('fill_bothub', help='Insert data from a source to Bothub') task_fill_bothub.add_argument('--repository', help='repository uuid destination on bothub') +task_fill_bothub.add_argument('--ownernick', help='Owner nickname') +task_fill_bothub.add_argument('--slug', help='Repository slug') +task_fill_bothub.add_argument('--lang', help='language of the bot on Bothub') # task_fill_bothub.add_argument('--source', help='path to the source file, e.g. intents.json') -task_fill_bothub.add_argument('--intent', help='name of the entity that represents the intents') task_fill_bothub.add_argument('--filename', help='name of the file that contains the phrases to be trained') task_fill_bothub.add_argument('--testfilename', help='name of the file that contains the phrases to be tested') task_fill_bothub.set_defaults(func=fill_bothub) @@ -37,14 +42,19 @@ def fill_bothub(args): def predict(args): phrases_count = 0 sum_bothub_hits = 0 + sum_bothub_fails = 0 expressions = load_json_file(args.testfilename) for expression in expressions['data']: phrases_count += 1 try: - bothub_result = bothub.analyze(expression['value'], args.lang) - if bothub_result['answer']['intent']['name'] == expression[args.intent]: + bothub_result = bothub.analyze_text(expression['value'], args.lang, args.ownernick, args.slug)#.analyze(expression['value'], args.lang) + # print(bothub_result['answer']['intent']['name']) + if bothub_result['answer']['intent']['name'] == expression['intent']: sum_bothub_hits += 1 print(expression['value']+' '+expression['intent']+' - OK') #+bothub_result['answer']['intent']['confidence'] + else: + sum_bothub_fails += 1 + print(expression['value']+' '+expression['intent']+' - FAILURE') except KeyError: traceback.print_exc() print('Skipping expression {} due to an error'.format(expression)) @@ -56,9 +66,25 @@ def predict(args): print('Bothub:') print('Analized phrases: {}'.format(phrases_count)) print('Correct predictions: {}'.format(sum_bothub_hits)) + print('Wrong predictions: {}'.format(sum_bothub_fails)) # print('Final Accuracy: {}'.format(bothub_accuracy)) # print('Average Confidence: {}%'.format(bothub_confidence_avg)) +def open_repository(args): + create_new_repository(args) + +task_predict = sub_tasks.add_parser('open_repository', help='Predict from Bothub and check accuracy') +task_predict.set_defaults(func=open_repository) + +def remove_repository(args): + delete_repository(args.ownernick, args.slug) + +task_train = sub_tasks.add_parser('remove_repository', help='Train all data uploaded to Bothub') +task_train.add_argument('--slug', help='slog fromt he repository') +task_train.add_argument('--ownernick', help='Owner nickname') +task_train.set_defaults(func=remove_repository) + + task_predict = sub_tasks.add_parser('predict', help='Predict from Bothub and check accuracy') # task_predict.add_argument('--authorization-bothub', help='authorization token from dataset on Bothub') diff --git a/benchmark/nlu_benchmark/services/app/bothub.py b/benchmark/nlu_benchmark/services/app/bothub.py index d71c996..f0f2944 100644 --- a/benchmark/nlu_benchmark/services/app/bothub.py +++ b/benchmark/nlu_benchmark/services/app/bothub.py @@ -6,11 +6,30 @@ user_token_header = {'Authorization': BOTHUB_USER_TOKEN} nlp_authorization_header = {'Authorization': BOTHUB_NLP_TOKEN} +def create_new_repository(args): + data = { + 'description':'temp_', + 'is_private': True, + 'language':'pt_br', + 'name':'temp_validation_1337_0g38h7', + 'slug':'temp_1337_0g38h7', + 'categories':[1]} + response = requests.post('{}/api/repository/new/'.format(BOTHUB_APP_URL),headers=user_token_header,data=data) + response_json = json.loads(response.text) + return [response_json["uuid"],response_json["slug"]] + + +def delete_repository(owner_nickname, repository_slug): + params = {'owner__nickname': owner_nickname, 'slug': repository_slug} + response = requests.delete('{0}/api/repository/{1}/{2}/'.format(BOTHUB_APP_URL, owner_nickname, repository_slug),headers=user_token_header,params=params) + print('Removed') + -def save_on_bothub(args, text, intent): - data = {'repository': args.repository, 'text': text, 'entities': [], 'intent': intent} - response = requests.post('{0}/api/example/new/'.format(BOTHUB_APP_URL), headers=user_token_header, data=data) - print('--> '+text+' {}'.format(response)) +def save_on_bothub(args, text, entities, intent): + data = {'repository': args, 'text': text, 'entities': entities, 'intent': intent} + response = requests.post('{0}/api/example/new/'.format(BOTHUB_APP_URL), headers=user_token_header, json=json.loads(json.dumps(data))) + #print('--> '+text+' {}'.format(response)) + print(response) def analyze(text, language): @@ -19,11 +38,18 @@ def analyze(text, language): response_json = json.loads(response.content) return response_json + +def analyze_text(text, language, owner_nickname, repository_slug): + data = {'text': text, 'language':language} + response = requests.post('{0}/api/repository/{1}/{2}/analyze/'.format(BOTHUB_APP_URL, owner_nickname, repository_slug), headers=user_token_header, data=data) + response_json = json.loads(response.content) + return response_json + def train(owner_nickname, repository_slug): params = {'owner__nickname': owner_nickname, 'slug': repository_slug} - response = requests.get('{0}/api/repository/{1}/{2}/train/'.format(BOTHUB_APP_URL,owner_nickname, repository_slug), params=params, headers=user_token_header) - print("Done") + requests.get('{0}/api/repository/{1}/{2}/train/'.format(BOTHUB_APP_URL,owner_nickname, repository_slug), params=params, headers=user_token_header) + print("Trained") def get_intent_data(result): diff --git a/benchmark/nlu_benchmark/services/app/utils.py b/benchmark/nlu_benchmark/services/app/utils.py index 99e5cbc..c8c88bd 100644 --- a/benchmark/nlu_benchmark/services/app/utils.py +++ b/benchmark/nlu_benchmark/services/app/utils.py @@ -1,11 +1,14 @@ import json +import io def load_json_file(entity_file): - with open(entity_file) as json_file: - data = json.load(json_file) - return data + # with open(entity_file) as json_file: + # data = json.dumps(json_file.read()) + # data_dict = json.loads(data) + data_dict = json.load(io.open(entity_file, 'r', encoding='utf-8-sig')) + return data_dict def percentage(part, whole): - return 100 * float(part) / float(whole) + return 100 * float(part) / float(whole) \ No newline at end of file From f925cab95bd279f44a91b3cf610be47b19247579 Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Tue, 31 Jul 2018 17:58:43 -0300 Subject: [PATCH 04/23] Add .editorconfig --- benchmark/.editorconfig | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 benchmark/.editorconfig diff --git a/benchmark/.editorconfig b/benchmark/.editorconfig new file mode 100644 index 0000000..3c44241 --- /dev/null +++ b/benchmark/.editorconfig @@ -0,0 +1,9 @@ +root = true + +[*] +indent_style = space +indent_size = 4 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true From 792eb06ca866e92426ab07fe0f778fbef773552d Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Tue, 31 Jul 2018 17:59:53 -0300 Subject: [PATCH 05/23] Add global config file and yaml support --- benchmark/Pipfile | 1 + benchmark/Pipfile.lock | 19 ++++++++++++++++++- benchmark/nlu_benchmark/cli/tools/bothub.py | 3 ++- benchmark/nlu_benchmark/services/bothub.py | 14 ++++++-------- benchmark/nlu_benchmark/services/service.py | 14 +++++++++++++- 5 files changed, 40 insertions(+), 11 deletions(-) diff --git a/benchmark/Pipfile b/benchmark/Pipfile index aaa0dc8..51cbc3a 100644 --- a/benchmark/Pipfile +++ b/benchmark/Pipfile @@ -6,6 +6,7 @@ name = "pypi" [packages] plac = "*" requests = "*" +pyyaml = "*" [dev-packages] diff --git a/benchmark/Pipfile.lock b/benchmark/Pipfile.lock index 5d6e25e..b460e77 100644 --- a/benchmark/Pipfile.lock +++ b/benchmark/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "7e83e40d821e8d569bf551375c17fe86300ff929c0808101ea4f27b8e5380acb" + "sha256": "1e451442ef616448ae0faefd5c0d7073e2755e7629c9cc68b3ebc57710a01d7a" }, "pipfile-spec": 6, "requires": { @@ -45,6 +45,23 @@ "index": "pypi", "version": "==0.9.6" }, + "pyyaml": { + "hashes": [ + "sha256:3d7da3009c0f3e783b2c873687652d83b1bbfd5c88e9813fb7e5b03c0dd3108b", + "sha256:3ef3092145e9b70e3ddd2c7ad59bdd0252a94dfe3949721633e41344de00a6bf", + "sha256:40c71b8e076d0550b2e6380bada1f1cd1017b882f7e16f09a65be98e017f211a", + "sha256:558dd60b890ba8fd982e05941927a3911dc409a63dcb8b634feaa0cda69330d3", + "sha256:a7c28b45d9f99102fa092bb213aa12e0aaf9a6a1f5e395d36166639c1f96c3a1", + "sha256:aa7dd4a6a427aed7df6fb7f08a580d68d9b118d90310374716ae90b710280af1", + "sha256:bc558586e6045763782014934bfaf39d48b8ae85a2713117d16c39864085c613", + "sha256:d46d7982b62e0729ad0175a9bc7e10a566fc07b224d2c79fafb5e032727eaa04", + "sha256:d5eef459e30b09f5a098b9cea68bebfeb268697f78d647bd255a085371ac7f3f", + "sha256:e01d3203230e1786cd91ccfdc8f8454c8069c91bee3962ad93b87a4b2860f537", + "sha256:e170a9e6fcfd19021dd29845af83bb79236068bf5fd4df3327c1be18182b2531" + ], + "index": "pypi", + "version": "==3.13" + }, "requests": { "hashes": [ "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", diff --git a/benchmark/nlu_benchmark/cli/tools/bothub.py b/benchmark/nlu_benchmark/cli/tools/bothub.py index a08f783..94b6f97 100644 --- a/benchmark/nlu_benchmark/cli/tools/bothub.py +++ b/benchmark/nlu_benchmark/cli/tools/bothub.py @@ -13,4 +13,5 @@ def get_token(username=None, password=None): username = input('username: ') if not password: password = getpass('password: ') - print(Bothub.login(username, password)) + user_token = Bothub.login(username, password) + print(f'Your user token is: {user_token}') diff --git a/benchmark/nlu_benchmark/services/bothub.py b/benchmark/nlu_benchmark/services/bothub.py index 9c57b7f..cccc744 100644 --- a/benchmark/nlu_benchmark/services/bothub.py +++ b/benchmark/nlu_benchmark/services/bothub.py @@ -4,6 +4,8 @@ class Bothub(Service): + GLOBAL_CONFIG_FILE = '.bothub-service.yaml' + @classmethod def api_request(cls, path, data=None): session = Session() @@ -21,11 +23,7 @@ def login(cls, username, password): response = cls.api_request( 'login', {'username': username, 'password': password}) - try: - response.raise_for_status() - return response.json().get('token') - except Exception as e: - raise Exception(' / '.join([ - f"{field}: {'; '.join(errors)}" for field, errors in - response.json().items() - ])) \ No newline at end of file + response.raise_for_status() + token = response.json().get('token') + cls.update_global_config(user_token=token) + return token diff --git a/benchmark/nlu_benchmark/services/service.py b/benchmark/nlu_benchmark/services/service.py index 938f7b7..ccd8896 100644 --- a/benchmark/nlu_benchmark/services/service.py +++ b/benchmark/nlu_benchmark/services/service.py @@ -1,2 +1,14 @@ +import os +import yaml + + class Service: - pass \ No newline at end of file + @classmethod + def update_global_config(cls, **config): + current_config = {} + if os.path.isfile(cls.GLOBAL_CONFIG_FILE): + stream = open(cls.GLOBAL_CONFIG_FILE, 'r') + current_config = yaml.load(stream) + current_config.update(config) + stream = open(cls.GLOBAL_CONFIG_FILE, 'w+') + stream.write(yaml.dump(current_config, default_flow_style=False)) \ No newline at end of file From 1069988bf838b543c5a0bbb7f5682dc291c1dfcb Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Tue, 31 Jul 2018 18:00:33 -0300 Subject: [PATCH 06/23] Add .gitignore and LICENSE --- benchmark/.gitignore | 2 + benchmark/LICENSE | 674 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 676 insertions(+) create mode 100644 benchmark/.gitignore create mode 100644 benchmark/LICENSE diff --git a/benchmark/.gitignore b/benchmark/.gitignore new file mode 100644 index 0000000..aac91ef --- /dev/null +++ b/benchmark/.gitignore @@ -0,0 +1,2 @@ +*-mockup.txt +.bothub-service.yaml diff --git a/benchmark/LICENSE b/benchmark/LICENSE new file mode 100644 index 0000000..94a9ed0 --- /dev/null +++ b/benchmark/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. From a752816404f29551966af730fdbb6b5933647a56 Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Tue, 31 Jul 2018 19:16:23 -0300 Subject: [PATCH 07/23] Add run to nlu_benchmark.cli --- benchmark/datafiles/binary.yaml | 14 ++++ benchmark/nlu_benchmark/cli/__main__.py | 4 +- benchmark/nlu_benchmark/cli/run.py | 36 +++++++++ benchmark/nlu_benchmark/services/bothub.py | 81 ++++++++++++++++++++- benchmark/nlu_benchmark/services/service.py | 9 ++- 5 files changed, 138 insertions(+), 6 deletions(-) create mode 100644 benchmark/datafiles/binary.yaml create mode 100644 benchmark/nlu_benchmark/cli/run.py diff --git a/benchmark/datafiles/binary.yaml b/benchmark/datafiles/binary.yaml new file mode 100644 index 0000000..ec70fae --- /dev/null +++ b/benchmark/datafiles/binary.yaml @@ -0,0 +1,14 @@ +language: en +train: + - text: "yes" + intent: "affirmative" + - text: "yeap" + intent: "affirmative" + - text: "okay" + intent: "affirmative" + - text: "no" + intent: "negative" + - text: "nop" + intent: "negative" + - text: "not" + intent: "negative" diff --git a/benchmark/nlu_benchmark/cli/__main__.py b/benchmark/nlu_benchmark/cli/__main__.py index 8fe82ae..7105888 100644 --- a/benchmark/nlu_benchmark/cli/__main__.py +++ b/benchmark/nlu_benchmark/cli/__main__.py @@ -2,10 +2,12 @@ import plac from .tools.bothub import get_token as bothub_get_token +from .run import run commands = { 'bothub_get_token': bothub_get_token, + 'run': run, } @plac.annotations( @@ -13,4 +15,4 @@ def main(command, *args): plac.call(commands.get(command), args) -plac.call(main, sys.argv[1:]) \ No newline at end of file +plac.call(main, sys.argv[1:]) diff --git a/benchmark/nlu_benchmark/cli/run.py b/benchmark/nlu_benchmark/cli/run.py new file mode 100644 index 0000000..c1f93a2 --- /dev/null +++ b/benchmark/nlu_benchmark/cli/run.py @@ -0,0 +1,36 @@ +import os +import plac +import yaml +import requests + +from ..services.bothub import Bothub + + +@plac.annotations( + data_file_path=plac.Annotation(), + bothub_user_token=plac.Annotation(kind='option')) +def run(data_file_path, bothub_user_token=None): + if not os.path.isfile(data_file_path): + return print(f'{data_file_path} not exists') + + data_file = open(data_file_path, 'r') + data = yaml.load(data_file) + + bothub_global_config = Bothub.get_current_global_config() + + bothub_user_token = bothub_user_token or bothub_global_config.get('user_token') + assert bothub_user_token + + bothub = Bothub(user_token=bothub_user_token) + create_repository_response = bothub.create_temporary_repository( + data.get('language')) + try: + create_repository_response.raise_for_status() + except requests.exceptions.HTTPError as e: + return print('Temporary repository not created :(\n' + + str(create_repository_response.content)) + temporary_repository = create_repository_response.json() + delete_repository_response = bothub.delete_repository( + temporary_repository.get('owner__nickname'), + temporary_repository.get('slug')) + delete_repository_response.raise_for_status() diff --git a/benchmark/nlu_benchmark/services/bothub.py b/benchmark/nlu_benchmark/services/bothub.py index cccc744..8c56d39 100644 --- a/benchmark/nlu_benchmark/services/bothub.py +++ b/benchmark/nlu_benchmark/services/bothub.py @@ -1,3 +1,6 @@ +import random +import string + from requests import Request, Session from .service import Service @@ -7,13 +10,19 @@ class Bothub(Service): GLOBAL_CONFIG_FILE = '.bothub-service.yaml' @classmethod - def api_request(cls, path, data=None): + def api_request(cls, path, data=None, headers={}, user_token=None, + replace_method=None): + if user_token: + headers.update({ 'Authorization': f'Token {user_token}' }) session = Session() + method = 'POST' if data else 'GET' + if replace_method: + method = replace_method request = Request( - 'POST' if data else 'GET', + method, f'https://bothub.it/api/{path}/', data=data, - headers={}) + headers=headers) prepped = request.prepare() return session.send(prepped) @@ -27,3 +36,69 @@ def login(cls, username, password): token = response.json().get('token') cls.update_global_config(user_token=token) return token + + @classmethod + def repository_exists(cls, owner_nickname, repository_slug): + response = Bothub.api_request( + f'repository/{owner_nickname}/{repository_slug}') + return response.status_code is 200 + + @classmethod + def get_categories(cls): + response = cls.api_request('categories') + response.raise_for_status() + return response.json() + + def __init__(self, user_token=None, **kwargs): + self.user_token = user_token + self._user_info = None + super().__init__(**kwargs) + + @property + def user_info(self): + if not self._user_info: + self._user_info = self.get_user_info() + return self._user_info + + def get_user_info(self): + assert self.user_token + response = Bothub.api_request( + 'my-profile', + user_token=self.user_token) + response.raise_for_status() + return response.json() + + def create_repository(self, **data): + assert self.user_token + return Bothub.api_request( + 'repository/new', + data=data, + user_token=self.user_token) + + def create_temporary_repository(self, language): + owner_nickname = self.user_info.get('nickname') + repository_slug = None + + while True: + repository_slug = ''.join([ + random.choice(string.ascii_uppercase + string.digits) + for _ in range(16) + ]) + if not Bothub.repository_exists(owner_nickname, repository_slug): + break + + categories = Bothub.get_categories() + + return self.create_repository(**{ + 'name': f'Temporary repository {repository_slug}', + 'slug': repository_slug, + 'language': language, + 'categories': map(lambda c: c.get('id'), categories[:1]), + 'description': 'This is a temporary repository to benchmark test.', + 'private': True, + }) + + def delete_repository(self, owner_nickname, repository_slug): + return Bothub.api_request( + f'repository/{owner_nickname}/{repository_slug}', + replace_method='DELETE') diff --git a/benchmark/nlu_benchmark/services/service.py b/benchmark/nlu_benchmark/services/service.py index ccd8896..b4c225b 100644 --- a/benchmark/nlu_benchmark/services/service.py +++ b/benchmark/nlu_benchmark/services/service.py @@ -4,11 +4,16 @@ class Service: @classmethod - def update_global_config(cls, **config): + def get_current_global_config(cls): current_config = {} if os.path.isfile(cls.GLOBAL_CONFIG_FILE): stream = open(cls.GLOBAL_CONFIG_FILE, 'r') current_config = yaml.load(stream) + return current_config + + @classmethod + def update_global_config(cls, **config): + current_config = cls.get_current_global_config() current_config.update(config) stream = open(cls.GLOBAL_CONFIG_FILE, 'w+') - stream.write(yaml.dump(current_config, default_flow_style=False)) \ No newline at end of file + stream.write(yaml.dump(current_config, default_flow_style=False)) From a3001a586c436f7d8f096b83a66cb4288bf8735a Mon Sep 17 00:00:00 2001 From: periclesJ Date: Thu, 2 Aug 2018 11:20:02 -0300 Subject: [PATCH 08/23] updates on main --- .../nlu_benchmark/services/app/__main__.py | 25 +- .../nlu_benchmark/services/app/bothub.py | 44 +- benchmark/nlu_benchmark/services/app/wit.py | 11 +- .../nlu_benchmark/services/testPhrases.json | 404 ++++++ .../nlu_benchmark/services/trainPhrases.json | 1109 +++++++++++++++++ 5 files changed, 1561 insertions(+), 32 deletions(-) create mode 100644 benchmark/nlu_benchmark/services/testPhrases.json create mode 100644 benchmark/nlu_benchmark/services/trainPhrases.json diff --git a/benchmark/nlu_benchmark/services/app/__main__.py b/benchmark/nlu_benchmark/services/app/__main__.py index 770a5ac..635d237 100644 --- a/benchmark/nlu_benchmark/services/app/__main__.py +++ b/benchmark/nlu_benchmark/services/app/__main__.py @@ -2,7 +2,7 @@ import traceback from app import bothub -from app.bothub import save_on_bothub, analyze, analyze_text, train, create_new_repository, delete_repository +from app.bothub import save_on_bothub, store_result, analyze_text, train, create_new_repository, delete_repository, write_csv_file from app.utils import load_json_file#,percentage parser = argparse.ArgumentParser(description='Train and Test the accuracy from Bothub') @@ -13,6 +13,7 @@ def fill_bothub(args): expressions = load_json_file(args.filename) repository_data = create_new_repository(args) + print('Uploading examples...') for n, expression in enumerate(expressions['data']): entities = [] if (len(expression['entities']) > 0): @@ -24,22 +25,23 @@ def fill_bothub(args): print('Skipping expression {} due to an error'.format(expression)) train_repository(args) + print('Generating report...') predict(args) delete_repository(args.ownernick, repository_data[1]) -# Add another argument that stores a description of the test + task_fill_bothub = sub_tasks.add_parser('fill_bothub', help='Insert data from a source to Bothub') task_fill_bothub.add_argument('--repository', help='repository uuid destination on bothub') task_fill_bothub.add_argument('--ownernick', help='Owner nickname') task_fill_bothub.add_argument('--slug', help='Repository slug') task_fill_bothub.add_argument('--lang', help='language of the bot on Bothub') -# task_fill_bothub.add_argument('--source', help='path to the source file, e.g. intents.json') task_fill_bothub.add_argument('--filename', help='name of the file that contains the phrases to be trained') task_fill_bothub.add_argument('--testfilename', help='name of the file that contains the phrases to be tested') task_fill_bothub.set_defaults(func=fill_bothub) def predict(args): + treated_predicts = [] phrases_count = 0 sum_bothub_hits = 0 sum_bothub_fails = 0 @@ -47,26 +49,18 @@ def predict(args): for expression in expressions['data']: phrases_count += 1 try: - bothub_result = bothub.analyze_text(expression['value'], args.lang, args.ownernick, args.slug)#.analyze(expression['value'], args.lang) - # print(bothub_result['answer']['intent']['name']) + bothub_result = bothub.analyze_text(expression['value'], args.lang, args.ownernick, args.slug) + treated_predicts.append(store_result(expression,bothub_result)) if bothub_result['answer']['intent']['name'] == expression['intent']: sum_bothub_hits += 1 - print(expression['value']+' '+expression['intent']+' - OK') #+bothub_result['answer']['intent']['confidence'] else: sum_bothub_fails += 1 - print(expression['value']+' '+expression['intent']+' - FAILURE') except KeyError: traceback.print_exc() print('Skipping expression {} due to an error'.format(expression)) - - print('============================ RESULT ================================') # bothub_accuracy = percentage(sum_bothub_hits, count) # bothub_confidence_avg = sum_bothub_confidence/count - - print('Bothub:') - print('Analized phrases: {}'.format(phrases_count)) - print('Correct predictions: {}'.format(sum_bothub_hits)) - print('Wrong predictions: {}'.format(sum_bothub_fails)) + write_csv_file(treated_predicts,'Analized phrases: {0}\nCorrect predictions: {1}\nWrong predictions: {2}'.format(phrases_count,sum_bothub_hits,sum_bothub_fails)) # print('Final Accuracy: {}'.format(bothub_accuracy)) # print('Average Confidence: {}%'.format(bothub_confidence_avg)) @@ -85,10 +79,7 @@ def remove_repository(args): task_train.set_defaults(func=remove_repository) - task_predict = sub_tasks.add_parser('predict', help='Predict from Bothub and check accuracy') -# task_predict.add_argument('--authorization-bothub', help='authorization token from dataset on Bothub') -# task_predict.add_argument('--source', help='path to the source file, e.g. intents.json') task_predict.add_argument('--testfilename', help='name of the source file') task_predict.add_argument('--intent', help='name of the entity that represents the intents') task_predict.add_argument('--lang', help='language of the bot on Bothub') diff --git a/benchmark/nlu_benchmark/services/app/bothub.py b/benchmark/nlu_benchmark/services/app/bothub.py index f0f2944..19fadde 100644 --- a/benchmark/nlu_benchmark/services/app/bothub.py +++ b/benchmark/nlu_benchmark/services/app/bothub.py @@ -6,6 +6,12 @@ user_token_header = {'Authorization': BOTHUB_USER_TOKEN} nlp_authorization_header = {'Authorization': BOTHUB_NLP_TOKEN} +valid_predict = [] +invalid_predict = [] +phrases_count = 0 +sum_bothub_hits = 0 +sum_bothub_fails = 0 + def create_new_repository(args): data = { 'description':'temp_', @@ -22,21 +28,14 @@ def create_new_repository(args): def delete_repository(owner_nickname, repository_slug): params = {'owner__nickname': owner_nickname, 'slug': repository_slug} response = requests.delete('{0}/api/repository/{1}/{2}/'.format(BOTHUB_APP_URL, owner_nickname, repository_slug),headers=user_token_header,params=params) + print('Saved output.csv') print('Removed') def save_on_bothub(args, text, entities, intent): data = {'repository': args, 'text': text, 'entities': entities, 'intent': intent} response = requests.post('{0}/api/example/new/'.format(BOTHUB_APP_URL), headers=user_token_header, json=json.loads(json.dumps(data))) - #print('--> '+text+' {}'.format(response)) - print(response) - - -def analyze(text, language): - data = {'text': text} - response = requests.post(BOTHUB_NLP_URL, headers=nlp_authorization_header, data=data) - response_json = json.loads(response.content) - return response_json + # print(response) def analyze_text(text, language, owner_nickname, repository_slug): @@ -46,10 +45,35 @@ def analyze_text(text, language, owner_nickname, repository_slug): return response_json +def store_result(expression, bothub_result): + global phrases_count + global sum_bothub_hits + global sum_bothub_fails + phrases_count += 1 + if bothub_result['answer']['intent']['name'] == expression['intent']: + sum_bothub_hits += 1 + return '{0},{1},{2},{3}%,OK'.format(expression['value'], expression['intent'], bothub_result['answer']['intent']['name'],int(float(bothub_result['answer']['intent']['confidence'])*100)) + else: + sum_bothub_fails += 1 + return '{0},{1},{2},{3}%,FAILURE'.format(expression['value'], expression['intent'], bothub_result['answer']['intent']['name'],int(float(bothub_result['answer']['intent']['confidence'])*100)) + #valid_predict.extend(invalid_predict) + #write_csv_file(valid_predict,'Analized phrases: {0}\nCorrect predictions: {1}\nWrong predictions: {2}'.format(phrases_count,sum_bothub_hits,sum_bothub_fails)) + + def train(owner_nickname, repository_slug): params = {'owner__nickname': owner_nickname, 'slug': repository_slug} requests.get('{0}/api/repository/{1}/{2}/train/'.format(BOTHUB_APP_URL,owner_nickname, repository_slug), params=params, headers=user_token_header) - print("Trained") + print("Bot trained") + + +def write_csv_file(data, csv_footer): + csv_headers = "Phrases,Expected intents,Bothub predicts,Confidence,Result\n" + with open('output1.csv', "w") as csv_file: + csv_file.write(csv_headers) + for line in data: + csv_file.write(line) + csv_file.write('\n') + csv_file.write('\n' + csv_footer) def get_intent_data(result): diff --git a/benchmark/nlu_benchmark/services/app/wit.py b/benchmark/nlu_benchmark/services/app/wit.py index 47253d0..992f7eb 100644 --- a/benchmark/nlu_benchmark/services/app/wit.py +++ b/benchmark/nlu_benchmark/services/app/wit.py @@ -1,8 +1,8 @@ import os - import requests from app.utils import load_json_file +from app.settings import WIT_APP_URL, WIT_TOKEN def get_expressions_data(source): @@ -34,7 +34,8 @@ def get_intent_from_input(intent, file_item): def analyze(text, token): - params = {'q': text} - headers = {'Authorization': token} - response = requests.get('https://api.wit.ai/message', params=params, headers=headers) - return response.json() + params = {'q':text} + headers = {'Authorization':WIT_TOKEN} + response = requests.get(WIT_APP_URL, params=params, headers=headers) + response_json = json.loads(response.content) + return response_json diff --git a/benchmark/nlu_benchmark/services/testPhrases.json b/benchmark/nlu_benchmark/services/testPhrases.json new file mode 100644 index 0000000..6e85786 --- /dev/null +++ b/benchmark/nlu_benchmark/services/testPhrases.json @@ -0,0 +1,404 @@ +{ + "data": [ + { + "value":"O Real Madrid jogará hoje?", + "intent":"esporte" + }, + { + "value":"Em que colocação está o Botafogo?", + "intent":"esporte" + }, + { + "value":"Quais partidas da rodada atual serão televisionadas?", + "intent":"esporte" + }, + { + "value":"Quantos gols o Barcelona fez no Liverpool?", + "intent":"esporte" + }, + { + "value":"Quem é o favorito do brasileirão?", + "intent":"esporte" + }, + { + "value":"Neymar está jogando em que time?", + "intent":"esporte" + }, + { + "value":"Qual time revelou o jogador Firmino?", + "intent":"esporte" + }, + { + "value":"Quando vai acontecer o próximo amistoso da seleção brasileira?", + "intent":"esporte" + }, + { + "value":"Quem foi o maior goleador da história?", + "intent":"esporte" + }, + { + "value":"Quem foi o melhor jogador do mundo Pelé ou Maradona?", + "intent":"esporte" + }, + { + "value":"Tem alguma lanchonete aberta?", + "intent":"comida" + }, + { + "value":"Qual o fastfood mais próximo da minha casa?", + "intent":"comida" + }, + { + "value":"Qual o cardápio do restaurante Divina's?", + "intent":"comida" + }, + { + "value":"Onde posso comer hamburguer hoje?", + "intent":"comida" + }, + { + "value":"Qual a cafeteria menos lotada na sexta feira?", + "intent":"comida" + }, + { + "value":"qual a faixa de preço do gbarbosa?", + "intent":"comida" + }, + { + "value":"O Careca abre segunda de 19:00 horas?", + "intent":"comida" + }, + { + "value":"Quanto custa a pizza de frango?", + "intent":"comida" + }, + { + "value":"Quais os melhores rodizios da cidade?", + "intent":"comida" + }, + { + "value":"Qual o delivery de sushi mais próximo da minha residência?", + "intent":"comida" + }, + { + "value":"Quais filmes de ação você me recomendaria?", + "intent":"cinema" + }, + { + "value":"Quem está estrelando uma noite no museu?", + "intent":"cinema" + }, + { + "value":"Quais os melhores documentários sobre astronomia?", + "intent":"cinema" + }, + { + "value":"Você já assistiu todo mundo em pânico?", + "intent":"cinema" + }, + { + "value":"Quais os clássicos de terror?", + "intent":"cinema" + }, + { + "value":"Quais os melhores filmes de steven spielberg?", + "intent":"cinema" + }, + { + "value":"Ainda existem locadoras de dvd?", + "intent":"cinema" + }, + { + "value":"Atores mais cotados para fazer Jurasic Park?", + "intent":"cinema" + }, + { + "value":"Brad pitt atuou em quantos filmes de drama?", + "intent":"cinema" + }, + { + "value":"Quando foi lançado desventuras em série?", + "intent":"cinema" + }, + { + "value":"Quantas pessoas nascem na indonésia a cada ano?", + "intent":"geografia" + }, + { + "value":"Qual o índice pluviométrico da carolina do norte?", + "intent":"geografia" + }, + { + "value":"Quantos países fazem fronteira com o Cazaquistão?", + "intent":"geografia" + }, + { + "value":"Qual a cidade mais quente da Nigéria?", + "intent":"geografia" + }, + { + "value":"Qual pais com maior volume populacional do mundo?", + "intent":"geografia" + }, + { + "value":"Qual é a área do Congo?", + "intent":"geografia" + }, + { + "value":"Vai chover hoje?", + "intent":"geografia" + }, + { + "value":"Quantas pessoas vivem hoje em Alagoas?", + "intent":"geografia" + }, + { + "value":"Qual a taxa de desemprego no Brasil?", + "intent":"geografia" + }, + { + "value":"Quantos continentes existem no planeta?", + "intent":"geografia" + }, + { + "value":"Preciso encontrar um bom psicólogo", + "intent":"saude" + }, + { + "value":"Qual médico eu deveria consultar para examinar os meus olhos?", + "intent":"saude" + }, + { + "value":"Estou sentindo uma forte dor nas costas o que devo fazer?", + "intent":"saude" + }, + { + "value":"Contraí uma gripe pesada devo tomar vitamina C?", + "intent":"saude" + }, + { + "value":"Meu filho está engasgado como devo proceder?", + "intent":"saude" + }, + { + "value":"Me feri com uma tesoura posso contrair tétano?", + "intent":"saude" + }, + { + "value":"Preciso me vacinar contra gripe.", + "intent":"saude" + }, + { + "value":"Onde fica o posto de saúde mais próximo?", + "intent":"saude" + }, + { + "value":"Existe alguma clínica 24h na minha cidade?", + "intent":"saude" + }, + { + "value":"Como posso descobrir se estou diabético?", + "intent":"saude" + }, + { + "value":"Eu posso pagar em uma lotérica?", + "intent":"financeiro" + }, + { + "value":"Posso pagar o boleto após o vencimento?", + "intent":"financeiro" + }, + { + "value":"Posso fazer adiantamento do valor devido?", + "intent":"financeiro" + }, + { + "value":"tem problema em efetuar o pagamento amanhã?", + "intent":"financeiro" + }, + { + "value":"É possível atualizar a data de vencimento?", + "intent":"financeiro" + }, + { + "value":"Eu gostaria de cancelar a minha compra", + "intent":"financeiro" + }, + { + "value":"Você poderia estornar a diferença da minha compra?", + "intent":"financeiro" + }, + { + "value":"Existe a possibilidade de financiar o restante da dívida?", + "intent":"financeiro" + }, + { + "value":"Eu já paguei", + "intent":"financeiro" + }, + { + "value":"Não quero mais receber essas cobranças", + "intent":"financeiro" + }, + { + "value":"Tô querendo moto modelo mais novo", + "intent":"automoveis" + }, + { + "value":"Compro ou troco carro usado", + "intent":"automoveis" + }, + { + "value":"Vende-se moto honda 150", + "intent":"automoveis" + }, + { + "value":"Quando sai o novo lançamento do FIAT", + "intent":"automoveis" + }, + { + "value":"Quero fazer manutenção no meu automóvel", + "intent":"automoveis" + }, + { + "value":"Troco carro uno 98 por moto", + "intent":"automoveis" + }, + { + "value":"Economizei para trocar de carro esse ano", + "intent":"automoveis" + }, + { + "value":"Compro carro que não seja caro", + "intent":"automoveis" + }, + { + "value":"Quero comprar o modelo atual do Civic", + "intent":"automoveis" + }, + { + "value":"Estou interessado na CB300", + "intent":"automoveis" + }, + { + "value":"Como faço pra votar?", + "intent":"politica" + }, + { + "value":"Posso escolher mais de um presidente?", + "intent":"politica" + }, + { + "value":"Tenho dúvida quanto ao canditado Péricles", + "intent":"politica" + }, + { + "value":"O candidato 15 é ladrão", + "intent":"politica" + }, + { + "value":"Vou votar em ninguém esse ano", + "intent":"politica" + }, + { + "value":"Meu voto será pro 51", + "intent":"politica" + }, + { + "value":"Prefiro o Partido Roxo", + "intent":"politica" + }, + { + "value":"Canditado 51 é ladrão vou votar no 15", + "intent":"politica" + }, + { + "value":"Qual o político menos ladrão pra votar?", + "intent":"politica" + }, + { + "value":"Peter para presidente do Brasil", + "intent":"politica" + }, + { + "value":"Tô afim de começar um jogo novo", + "intent":"games" + }, + { + "value":"Tô com task vou jogar só amanhã", + "intent":"games" + }, + { + "value":"Posso jogar com vocês?", + "intent":"games" + }, + { + "value":"Esse game pega Multiplayer", + "intent":"games" + }, + { + "value":"Dá pra logar no PUBG?", + "intent":"games" + }, + { + "value":"Tô esperando você ficar online pra gente jogar", + "intent":"games" + }, + { + "value":"Qual o seu game favorito?", + "intent":"games" + }, + { + "value":"No PC eu uso a Steam", + "intent":"games" + }, + { + "value":"Já jogou algum jogo da Blizzard?", + "intent":"games" + }, + { + "value":"Gosto de jogar Minecraft no celular", + "intent":"games" + }, + { + "value":"Como tratar as pulgas do meu Javalí?", + "intent":"pets" + }, + { + "value":"Meus cachorros não se dão bem", + "intent":"pets" + }, + { + "value":"Minha gata está vomitando muitas bolas de pelo", + "intent":"pets" + }, + { + "value":"Meu cachorro não está muito bem de saúde", + "intent":"pets" + }, + { + "value":"Estou criando um novo filhote de urso", + "intent":"pets" + }, + { + "value":"Meu porco de estimação está com pulgas", + "intent":"pets" + }, + { + "value":"Meus pets latem muito como resolver?", + "intent":"pets" + }, + { + "value":"Quanto tempo vive um gato egípcio?", + "intent":"pets" + }, + { + "value":"tenho muitos pets como cuidar de todos?", + "intent":"pets" + }, + { + "value":"posso criar um cachorro em apartamento?", + "intent":"pets" + } + ] +} \ No newline at end of file diff --git a/benchmark/nlu_benchmark/services/trainPhrases.json b/benchmark/nlu_benchmark/services/trainPhrases.json new file mode 100644 index 0000000..6884858 --- /dev/null +++ b/benchmark/nlu_benchmark/services/trainPhrases.json @@ -0,0 +1,1109 @@ +{ + "data": [ + { + "text": "Qual time vai jogar hoje?", + "intent": "esporte", + "entities": [] + }, + { + "text": "Quem é o primeiro colocado na tabela?", + "intent": "esporte", + "entities": [ + { + "entity": "ranking", + "start": 9, + "end": 26 + } + ] + }, + { + "text": "Quem venceu o campeonato do último ano?", + "intent": "esporte", + "entities": [ + { + "entity": "competicao", + "start": 14, + "end": 24 + }, + { + "entity": "ranking", + "start": 5, + "end": 11 + }, + { + "entity": "data", + "start": 28, + "end": 38 + } + ] + }, + { + "text": "Quais os times estão competindo pelo primeiro lugar?", + "intent": "esporte", + "entities": [ + { + "entity": "competicao", + "start": 21, + "end": 31 + }, + { + "entity": "ranking", + "start": 37, + "end": 51 + } + ] + }, + { + "text": "Com quantos pontos está o primeiro colocado?", + "intent": "esporte", + "entities": [ + { + "entity": "ranking", + "start": 26, + "end": 43 + } + ] + }, + { + "text": "Quem está na lanterna depois dessa última rodada?", + "intent": "esporte", + "entities": [] + }, + { + "text": "Quais jogos irão ocorrer durante essa semana?", + "intent": "esporte", + "entities": [ + { + "entity": "data", + "start": 33, + "end": 44 + } + ] + }, + { + "text": "Quem venceu a copa de 2002?", + "intent": "esporte", + "entities": [ + { + "entity": "ranking", + "start": 5, + "end": 11 + }, + { + "entity": "competicao", + "start": 14, + "end": 18 + }, + { + "entity": "data", + "start": 22, + "end": 26 + } + ] + }, + { + "text": "Como está a tabela dessa temporada?", + "intent": "esporte", + "entities": [] + }, + { + "text": "Qual a escalação da seleção brasileira?", + "intent": "esporte", + "entities": [ + { + "start": 20, + "end": 38, + "entity": "time" + } + ] + }, + { + "text": "O que você me recomendaria para jantar hoje?", + "intent": "comida", + "entities": [ + { + "start": 32, + "end": 38, + "entity": "refeicao" + } + ] + }, + { + "text": "Onde fica o restaurante mais próximo?", + "intent": "comida", + "entities": [ + { + "entity": "lugar", + "start": 12, + "end": 23 + }, + { + "entity": "lugar", + "start": 24, + "end": 36 + }, + { + "start": 12, + "end": 23, + "entity": "estabelecimento" + } + ] + }, + { + "text": "O que tem mais calorias, suco de frutas ou cerveja?", + "intent": "comida", + "entities": [ + { + "start": 25, + "end": 39, + "entity": "bebida" + }, + { + "start": 43, + "end": 50, + "entity": "bebida" + } + ] + }, + { + "text": "Onde eu posso comprar chocolate?", + "intent": "comida", + "entities": [ + { + "start": 22, + "end": 31, + "entity": "tipo_comida" + } + ] + }, + { + "text": "Comidas salgadas ou doces?", + "intent": "comida", + "entities": [ + { + "start": 8, + "end": 16, + "entity": "sabor" + }, + { + "start": 20, + "end": 25, + "entity": "sabor" + } + ] + }, + { + "text": "Você gosta de culinária estrangeira?", + "intent": "comida", + "entities": [ + { + "start": 14, + "end": 35, + "entity": "tipo_comida" + } + ] + }, + { + "text": "Qual a sua preferida, junk food ou health food?", + "intent": "comida", + "entities": [ + { + "start": 22, + "end": 31, + "entity": "tipo_comida" + }, + { + "start": 35, + "end": 46, + "entity": "tipo_comida" + } + ] + }, + { + "text": "Você prefere comer fora ou pedir delivery?", + "intent": "comida", + "entities": [ + { + "start": 19, + "end": 23, + "entity": "lugar" + }, + { + "start": 33, + "end": 41, + "entity": "tipo_comida" + } + ] + }, + { + "text": "Sua carne deve ser bem passada ou mal passada?", + "intent": "comida", + "entities": [ + { + "start": 4, + "end": 9, + "entity": "tipo_comida" + }, + { + "start": 19, + "end": 30, + "entity": "tipo_comida" + }, + { + "start": 34, + "end": 45, + "entity": "tipo_comida" + } + ] + }, + { + "text": "Qual seria o seu café da manhã ideal?", + "intent": "comida", + "entities": [ + { + "start": 17, + "end": 21, + "entity": "bebida" + }, + { + "start": 25, + "end": 30, + "entity": "data" + } + ] + }, + { + "text": "Qual é a melhor série de comédia já criada?", + "intent": "cinema", + "entities": [ + { + "start": 25, + "end": 32, + "entity": "genero" + }, + { + "start": 16, + "end": 21, + "entity": "tipo_de_programa" + } + ] + }, + { + "text": "Que filmes irão estreiar esse ano?", + "intent": "cinema", + "entities": [ + { + "start": 4, + "end": 10, + "entity": "tipo_de_programa" + }, + { + "start": 25, + "end": 34, + "entity": "data" + } + ] + }, + { + "text": "Quantos episódio tem breaking bad?", + "intent": "cinema", + "entities": [ + { + "start": 21, + "end": 33, + "entity": "serie" + } + ] + }, + { + "text": "Quais atores ganharam Oscar em 2017?", + "intent": "cinema", + "entities": [ + { + "start": 22, + "end": 27, + "entity": "premiacao" + }, + { + "start": 31, + "end": 35, + "entity": "data" + }, + { + "start": 6, + "end": 12, + "entity": "profissao" + } + ] + }, + { + "text": "Qual filme vendeu mais entradas de cinema na história?", + "intent": "cinema", + "entities": [ + { + "start": 5, + "end": 10, + "entity": "tipo_de_programa" + }, + { + "start": 11, + "end": 17, + "entity": "ranking" + } + ] + }, + { + "text": "Que atores participaram de Titanic?", + "intent": "cinema", + "entities": [ + { + "start": 27, + "end": 34, + "entity": "tipo_de_programa" + } + ] + }, + { + "text": "Qual gênero de cinema você me recomendaria?", + "intent": "cinema", + "entities": [] + }, + { + "text": "Salas de cinema são um bom lugar para marcar encontros?", + "intent": "cinema", + "entities": [ + { + "start": 0, + "end": 15, + "entity": "lugar" + } + ] + }, + { + "text": "Os diretores de cinema de hollywood são famosos?", + "intent": "cinema", + "entities": [ + { + "start": 26, + "end": 35, + "entity": "lugar" + }, + { + "start": 3, + "end": 12, + "entity": "profissao" + } + ] + }, + { + "text": "Qual é a nota de velozes e furiosos no imdb?", + "intent": "cinema", + "entities": [ + { + "start": 17, + "end": 35, + "entity": "filme" + } + ] + }, + { + "text": "Quantos estados tem o meu pais?", + "intent": "geografia", + "entities": [ + { + "start": 26, + "end": 30, + "entity": "lugar" + }, + { + "start": 8, + "end": 15, + "entity": "lugar" + } + ] + }, + { + "text": "Qual o perímetro do Brasil?", + "intent": "geografia", + "entities": [ + { + "start": 20, + "end": 26, + "entity": "pais" + } + ] + }, + { + "text": "Quantos estados nós temos na região sul?", + "intent": "geografia", + "entities": [ + { + "start": 8, + "end": 15, + "entity": "lugar" + } + ] + }, + { + "text": "Quantos habitantes tem a capital do méxico?", + "intent": "geografia", + "entities": [ + { + "start": 25, + "end": 32, + "entity": "lugar" + }, + { + "start": 36, + "end": 42, + "entity": "pais" + } + ] + }, + { + "text": "A que altitude fica média fica o equador?", + "intent": "geografia", + "entities": [ + { + "start": 33, + "end": 40, + "entity": "lugar" + }, + { + "start": 33, + "end": 40, + "entity": "pais" + } + ] + }, + { + "text": "Qual é o clima predominante na Europa?", + "intent": "geografia", + "entities": [ + { + "start": 31, + "end": 37, + "entity": "continente" + } + ] + }, + { + "text": "Qual a distância da China ao Canadá?", + "intent": "geografia", + "entities": [ + { + "start": 20, + "end": 25, + "entity": "pais" + }, + { + "start": 29, + "end": 35, + "entity": "pais" + } + ] + }, + { + "text": "Qual o índice de chuvas para abril?", + "intent": "geografia", + "entities": [ + { + "start": 29, + "end": 34, + "entity": "data" + }, + { + "start": 29, + "end": 34, + "entity": "mes" + }, + { + "start": 17, + "end": 23, + "entity": "clima" + } + ] + }, + { + "text": "Qual a chance de ocorrer trovões no deserto?", + "intent": "geografia", + "entities": [ + { + "start": 36, + "end": 43, + "entity": "lugar" + }, + { + "start": 25, + "end": 32, + "entity": "clima" + }, + { + "start": 36, + "end": 43, + "entity": "clima" + } + ] + }, + { + "text": "Qual o índice de natalidade do estado?", + "intent": "geografia", + "entities": [ + { + "start": 31, + "end": 37, + "entity": "lugar" + } + ] + }, + { + "text": "Estou com dor de cabeça, o que devo fazer?", + "intent": "saude", + "entities": [ + { + "start": 17, + "end": 23, + "entity": "parte_do_corpo" + } + ] + }, + { + "text": "Meu dente está doendo, qual pode ser o problema?", + "intent": "saude", + "entities": [ + { + "start": 4, + "end": 9, + "entity": "parte_do_corpo" + } + ] + }, + { + "text": "Qual é o melhor remédio para labirintite?", + "intent": "saude", + "entities": [ + { + "start": 29, + "end": 40, + "entity": "patologia" + } + ] + }, + { + "text": "Estou sentindo algumas tonturas, o que pode ser isso?", + "intent": "saude", + "entities": [ + { + "start": 23, + "end": 31, + "entity": "patologia" + } + ] + }, + { + "text": "Preciso de um antibiótico para minha faringite.", + "intent": "saude", + "entities": [ + { + "start": 14, + "end": 25, + "entity": "remedio" + }, + { + "start": 37, + "end": 46, + "entity": "patologia" + } + ] + }, + { + "text": "Qual drogaria possui maior diversidade de produtos?", + "intent": "saude", + "entities": [ + { + "start": 5, + "end": 13, + "entity": "estabelecimento" + } + ] + }, + { + "text": "O que você me recomendaria para tratar minha dermatite?", + "intent": "saude", + "entities": [ + { + "start": 45, + "end": 54, + "entity": "patologia" + } + ] + }, + { + "text": "Você possui algum spray para massagem?", + "intent": "saude", + "entities": [] + }, + { + "text": "Meu médico me receitou um antialérgico, você poderia verificar o valor?", + "intent": "saude", + "entities": [] + }, + { + "text": "Me sinto constipado.", + "intent": "saude", + "entities": [ + { + "start": 9, + "end": 19, + "entity": "patologia" + } + ] + }, + { + "text": "Eu possuo algum boleto em aberto?", + "intent": "financeiro", + "entities": [] + }, + { + "text": "Quantas parcelas ainda estão em atraso?", + "intent": "financeiro", + "entities": [] + }, + { + "text": "Onde eu posso efetuar o pagamento?", + "intent": "financeiro", + "entities": [] + }, + { + "text": "Você poderia enviar o valor total para mim?", + "intent": "financeiro", + "entities": [] + }, + { + "text": "Qual a data de vencimento da parcela atual?", + "intent": "financeiro", + "entities": [ + { + "start": 7, + "end": 25, + "entity": "data" + } + ] + }, + { + "text": "Meu nome está no spc ou no serasa?", + "intent": "financeiro", + "entities": [] + }, + { + "text": "Quanto de juros eu estou pagando?", + "intent": "financeiro", + "entities": [] + }, + { + "text": "Existe desconto para pagamentos a vista?", + "intent": "financeiro", + "entities": [] + }, + { + "text": "Eu gostaria de negociar minha dívida", + "intent": "financeiro", + "entities": [] + }, + { + "text": "Evite cobranças, quite suas dívidas com antecedência", + "intent": "financeiro", + "entities": [] + }, + { + "text": "Quais os modelos vocês estão vendendo?", + "intent": "automoveis", + "entities": [] + }, + { + "text": "Tô afim de trocar o meu carro", + "intent": "automoveis", + "entities": [ + { + "start": 24, + "end": 29, + "entity": "veiculo" + } + ] + }, + { + "text": "Quero comprar o carro do ano", + "intent": "automoveis", + "entities": [ + { + "start": 22, + "end": 28, + "entity": "data" + } + ] + }, + { + "text": "Vocês têm Fiat?", + "intent": "automoveis", + "entities": [ + { + "start": 10, + "end": 14, + "entity": "modelo_carro" + } + ] + }, + { + "text": "Só compro carro se for da Citroen", + "intent": "automoveis", + "entities": [ + { + "start": 26, + "end": 33, + "entity": "marca" + } + ] + }, + { + "text": "Desejo comprar carro barato", + "intent": "automoveis", + "entities": [] + }, + { + "text": "Me interesso por Moto Honda", + "intent": "automoveis", + "entities": [ + { + "start": 17, + "end": 21, + "entity": "veiculo" + }, + { + "start": 22, + "end": 27, + "entity": "marca" + } + ] + }, + { + "text": "Gostaria de uma indicação de automóvel", + "intent": "automoveis", + "entities": [] + }, + { + "text": "Posso alugar um Mustang?", + "intent": "automoveis", + "entities": [ + { + "start": 16, + "end": 23, + "entity": "modelo_carro" + } + ] + }, + { + "text": "Troco moto CB 600 amarela", + "intent": "automoveis", + "entities": [ + { + "start": 11, + "end": 17, + "entity": "modelo_moto" + }, + { + "start": 18, + "end": 25, + "entity": "cor" + } + ] + }, + { + "text": "Qual o melhor candidato desse ano?", + "intent": "politica", + "entities": [ + { + "start": 24, + "end": 33, + "entity": "data" + } + ] + }, + { + "text": "Quero votar no candidato menos corrupto", + "intent": "politica", + "entities": [] + }, + { + "text": "Quais serão os candidatos a prefeito esse ano?", + "intent": "politica", + "entities": [ + { + "start": 37, + "end": 45, + "entity": "data" + } + ] + }, + { + "text": "Eu vou votar no presidente Peter", + "intent": "politica", + "entities": [ + { + "start": 16, + "end": 26, + "entity": "cargo_politico" + } + ] + }, + { + "text": "Quero votar no Péricles para presidente", + "intent": "politica", + "entities": [] + }, + { + "text": "Em quem votar esse ano para presidência?", + "intent": "politica", + "entities": [ + { + "start": 28, + "end": 39, + "entity": "cargo_politico" + } + ] + }, + { + "text": "Vou votar em branco", + "intent": "politica", + "entities": [] + }, + { + "text": "Meu voto esse ano será nulo", + "intent": "politica", + "entities": [] + }, + { + "text": "acho que votarei no 15", + "intent": "politica", + "entities": [] + }, + { + "text": "Duvidoso entre votar no 15 ou no 51", + "intent": "politica", + "entities": [] + }, + { + "text": "Gosto de jogos de ação", + "intent": "games", + "entities": [] + }, + { + "text": "Você joga Fortinite?", + "intent": "games", + "entities": [ + { + "start": 10, + "end": 19, + "entity": "titulo_jogo" + } + ] + }, + { + "text": "Meu estilo preferido de jogo é de MMORPG", + "intent": "games", + "entities": [ + { + "start": 34, + "end": 40, + "entity": "tipo_jogo" + } + ] + }, + { + "text": "Prefiro jogar Forza Horizon", + "intent": "games", + "entities": [ + { + "start": 14, + "end": 27, + "entity": "titulo_jogo" + } + ] + }, + { + "text": "Meu game favorito é o Watch Dogs", + "intent": "games", + "entities": [ + { + "start": 22, + "end": 32, + "entity": "titulo_jogo" + } + ] + }, + { + "text": "Prefiro jogos para celular", + "intent": "games", + "entities": [ + { + "start": 8, + "end": 26, + "entity": "tipo_jogo" + } + ] + }, + { + "text": "Game bom pra mim só no PC", + "intent": "games", + "entities": [] + }, + { + "text": "Tenho interesse nos jogos da Blizzard", + "intent": "games", + "entities": [ + { + "start": 29, + "end": 37, + "entity": "marca" + } + ] + }, + { + "text": "Já viu os Jogos da E3?", + "intent": "games", + "entities": [ + { + "start": 19, + "end": 21, + "entity": "marca" + } + ] + }, + { + "text": "Gosto mais de FPS do que de MOBA", + "intent": "games", + "entities": [ + { + "start": 28, + "end": 32, + "entity": "tipo_jogo" + }, + { + "start": 14, + "end": 17, + "entity": "tipo_jogo" + } + ] + }, + { + "text": "O que fazer para acabar com pulgas e carrapatos?", + "intent": "pets", + "entities": [ + { + "start": 28, + "end": 34, + "entity": "patologia" + }, + { + "start": 37, + "end": 47, + "entity": "patologia" + }, + { + "start": 28, + "end": 34, + "entity": "animal" + }, + { + "start": 37, + "end": 47, + "entity": "animal" + } + ] + }, + { + "text": "com que frequência devo dar banho no meu gato?", + "intent": "pets", + "entities": [ + { + "start": 41, + "end": 45, + "entity": "animal" + } + ] + }, + { + "text": "Como diminuir a queda de pelos?", + "intent": "pets", + "entities": [ + { + "start": 16, + "end": 30, + "entity": "patologia" + } + ] + }, + { + "text": "Onde eu posso castrar meu gato?", + "intent": "pets", + "entities": [] + }, + { + "text": "Qual a ração mais indicada para filhotes?", + "intent": "pets", + "entities": [] + }, + { + "text": "Obtive 6 filhotes na última ninhada", + "intent": "pets", + "entities": [] + }, + { + "text": "cães e gatos também tem gripe?", + "intent": "pets", + "entities": [ + { + "start": 0, + "end": 4, + "entity": "animal" + }, + { + "start": 24, + "end": 29, + "entity": "patologia" + } + ] + }, + { + "text": "Qual o petshop mais confiável da cidade?", + "intent": "pets", + "entities": [ + { + "start": 33, + "end": 39, + "entity": "lugar" + }, + { + "start": 7, + "end": 14, + "entity": "estabelecimento" + } + ] + }, + { + "text": "meus animais são muito agressivos", + "intent": "pets", + "entities": [] + }, + { + "text": "contratei um adestrador mais meus bichos", + "intent": "pets", + "entities": [ + { + "start": 13, + "end": 23, + "entity": "profissao" + } + ] + } + ] +} \ No newline at end of file From 08d948b9c17188142a84e82669a717f85a7facc8 Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Fri, 3 Aug 2018 16:55:19 -0300 Subject: [PATCH 09/23] Move version 1, add logger --- benchmark/.gitignore | 1 + benchmark/Pipfile | 1 + benchmark/Pipfile.lock | 41 +- .../services => datafiles}/testPhrases.json | 0 .../services => datafiles}/trainPhrases.json | 0 benchmark/nlu_benchmark/cli/__init__.py | 4 + benchmark/nlu_benchmark/cli/__main__.py | 31 +- benchmark/nlu_benchmark/cli/run.py | 45 +- benchmark/nlu_benchmark/services/LICENSE | 674 ------------------ benchmark/nlu_benchmark/services/Pipfile | 13 - benchmark/nlu_benchmark/services/Pipfile.lock | 135 ---- benchmark/nlu_benchmark/services/__init__.py | 4 + .../services/app => v1}/__init__.py | 0 .../services/app => v1}/__main__.py | 0 .../services/app => v1}/bothub.py | 0 .../services/app => v1}/utils.py | 0 .../{nlu_benchmark/services/app => v1}/wit.py | 0 17 files changed, 110 insertions(+), 839 deletions(-) rename benchmark/{nlu_benchmark/services => datafiles}/testPhrases.json (100%) rename benchmark/{nlu_benchmark/services => datafiles}/trainPhrases.json (100%) delete mode 100644 benchmark/nlu_benchmark/services/LICENSE delete mode 100644 benchmark/nlu_benchmark/services/Pipfile delete mode 100644 benchmark/nlu_benchmark/services/Pipfile.lock rename benchmark/{nlu_benchmark/services/app => v1}/__init__.py (100%) rename benchmark/{nlu_benchmark/services/app => v1}/__main__.py (100%) rename benchmark/{nlu_benchmark/services/app => v1}/bothub.py (100%) rename benchmark/{nlu_benchmark/services/app => v1}/utils.py (100%) rename benchmark/{nlu_benchmark/services/app => v1}/wit.py (100%) diff --git a/benchmark/.gitignore b/benchmark/.gitignore index aac91ef..7c8d5b9 100644 --- a/benchmark/.gitignore +++ b/benchmark/.gitignore @@ -1,2 +1,3 @@ *-mockup.txt .bothub-service.yaml +*.log diff --git a/benchmark/Pipfile b/benchmark/Pipfile index 51cbc3a..04ba665 100644 --- a/benchmark/Pipfile +++ b/benchmark/Pipfile @@ -9,6 +9,7 @@ requests = "*" pyyaml = "*" [dev-packages] +"flake8" = "*" [requires] python_version = "3.6" diff --git a/benchmark/Pipfile.lock b/benchmark/Pipfile.lock index b460e77..4af1286 100644 --- a/benchmark/Pipfile.lock +++ b/benchmark/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "1e451442ef616448ae0faefd5c0d7073e2755e7629c9cc68b3ebc57710a01d7a" + "sha256": "2ebcdfec05ef7338ee9e78ac18c638e162baf944079a26f3fe5894aa2e2243e7" }, "pipfile-spec": 6, "requires": { @@ -39,11 +39,11 @@ }, "plac": { "hashes": [ - "sha256:854693ad90367e8267112ffbb8955f57d6fdeac3191791dc9ffce80f87fd2370", - "sha256:ba3f719a018175f0a15a6b04e6cc79c25fd563d348aacd320c3644d2a9baf89b" + "sha256:879d3009bee474cc96b5d7a4ebdf6fa0c4931008ecb858caf09eed9ca302c8da", + "sha256:b03f967f535b3bf5a71b191fa5eb09872a5cfb1e3b377efc4138995e10ba36d7" ], "index": "pypi", - "version": "==0.9.6" + "version": "==1.0.0" }, "pyyaml": { "hashes": [ @@ -75,9 +75,38 @@ "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" ], - "markers": "python_version != '3.0.*' and python_version < '4' and python_version != '3.2.*' and python_version != '3.1.*' and python_version != '3.3.*' and python_version >= '2.6'", "version": "==1.23" } }, - "develop": {} + "develop": { + "flake8": { + "hashes": [ + "sha256:7253265f7abd8b313e3892944044a365e3f4ac3fcdcfb4298f55ee9ddf188ba0", + "sha256:c7841163e2b576d435799169b78703ad6ac1bbb0f199994fc05f700b2a90ea37" + ], + "index": "pypi", + "version": "==3.5.0" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, + "pycodestyle": { + "hashes": [ + "sha256:682256a5b318149ca0d2a9185d365d8864a768a28db66a84a2ea946bcc426766", + "sha256:6c4245ade1edfad79c3446fadfc96b0de2759662dc29d07d80a6f27ad1ca6ba9" + ], + "version": "==2.3.1" + }, + "pyflakes": { + "hashes": [ + "sha256:08bd6a50edf8cffa9fa09a463063c425ecaaf10d1eb0335a7e8b1401aef89e6f", + "sha256:8d616a382f243dbf19b54743f280b80198be0bca3a5396f1d2e1fca6223e8805" + ], + "version": "==1.6.0" + } + } } diff --git a/benchmark/nlu_benchmark/services/testPhrases.json b/benchmark/datafiles/testPhrases.json similarity index 100% rename from benchmark/nlu_benchmark/services/testPhrases.json rename to benchmark/datafiles/testPhrases.json diff --git a/benchmark/nlu_benchmark/services/trainPhrases.json b/benchmark/datafiles/trainPhrases.json similarity index 100% rename from benchmark/nlu_benchmark/services/trainPhrases.json rename to benchmark/datafiles/trainPhrases.json diff --git a/benchmark/nlu_benchmark/cli/__init__.py b/benchmark/nlu_benchmark/cli/__init__.py index e69de29..3d9c189 100644 --- a/benchmark/nlu_benchmark/cli/__init__.py +++ b/benchmark/nlu_benchmark/cli/__init__.py @@ -0,0 +1,4 @@ +import logging + + +logger = logging.getLogger('nlu_benchmark.cli') diff --git a/benchmark/nlu_benchmark/cli/__main__.py b/benchmark/nlu_benchmark/cli/__main__.py index 7105888..af1fc78 100644 --- a/benchmark/nlu_benchmark/cli/__main__.py +++ b/benchmark/nlu_benchmark/cli/__main__.py @@ -1,18 +1,45 @@ import sys +import logging import plac +from . import logger as cli_logger +from ..services import logger as services_logger from .tools.bothub import get_token as bothub_get_token from .run import run +LOGGER_FORMAT = { + 'StreamHandler': '%(name)-22s - %(levelname)-8s - %(message)s', + 'FileHandler': '%(asctime)-15s - %(name)-22s - %(levelname)-8s - %(message)s' +} + commands = { 'bothub_get_token': bothub_get_token, 'run': run, } @plac.annotations( + debug=plac.Annotation(kind='flag', abbrev='D'), command=plac.Annotation(choices=commands.keys())) -def main(command, *args): - plac.call(commands.get(command), args) +def main(debug, command, *args): + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG if debug else logging.INFO) + ch.setFormatter(logging.Formatter(LOGGER_FORMAT.get('StreamHandler'))) + fh = logging.FileHandler('nlu_benchmark.log') + fh.setLevel(logging.DEBUG) + fh.setFormatter(logging.Formatter(LOGGER_FORMAT.get('FileHandler'))) + + for logger in [cli_logger, services_logger]: + logger.setLevel(logging.DEBUG) + logger.addHandler(fh) + logger.addHandler(ch) + + cli_logger.info(f'NLU Benchmark running {command} command') + + try: + plac.call(commands.get(command), args) + except Exception as e: + cli_logger.exception(e) + plac.call(main, sys.argv[1:]) diff --git a/benchmark/nlu_benchmark/cli/run.py b/benchmark/nlu_benchmark/cli/run.py index c1f93a2..19c3218 100644 --- a/benchmark/nlu_benchmark/cli/run.py +++ b/benchmark/nlu_benchmark/cli/run.py @@ -1,36 +1,63 @@ -import os import plac import yaml +import json import requests +from . import logger from ..services.bothub import Bothub @plac.annotations( data_file_path=plac.Annotation(), + data_file_type=plac.Annotation(kind='option', choices=['yaml', 'json']), bothub_user_token=plac.Annotation(kind='option')) -def run(data_file_path, bothub_user_token=None): - if not os.path.isfile(data_file_path): - return print(f'{data_file_path} not exists') +def run(data_file_path, data_file_type='yaml', bothub_user_token=None): + try: + data_file = open(data_file_path, 'r') + except Exception as e: + logger.error(f'Can\'t open the data file "{data_file_path}"') + raise e + + try: + if data_file_type == 'yaml': + data = yaml.load(data_file) + elif data_file_type == 'json': + data = json.load(data_file) + except Exception as e: + raise e - data_file = open(data_file_path, 'r') - data = yaml.load(data_file) + logger.debug(f'data loaded to run benchmark {data}') bothub_global_config = Bothub.get_current_global_config() bothub_user_token = bothub_user_token or bothub_global_config.get('user_token') assert bothub_user_token + logger.debug(f'using {bothub_user_token} bothub user token') + logger.info('Starting Bothub benchmark...') bothub = Bothub(user_token=bothub_user_token) + create_repository_response = bothub.create_temporary_repository( data.get('language')) + try: create_repository_response.raise_for_status() except requests.exceptions.HTTPError as e: - return print('Temporary repository not created :(\n' + - str(create_repository_response.content)) + logger.error('Bothub temporary repository not created') + logger.debug(create_repository_response.text) + raise e + temporary_repository = create_repository_response.json() + logger.info(f'Bothub temporary repository created "{temporary_repository.get("name")}"') + delete_repository_response = bothub.delete_repository( temporary_repository.get('owner__nickname'), temporary_repository.get('slug')) - delete_repository_response.raise_for_status() + + try: + delete_repository_response.raise_for_status() + except Exception as e: + logger.warning('Bothub temporary repository not deleted, manually delete') + logger.debug(delete_repository_response.text) + + logger.info(f'Bothub temporary repository deleted') diff --git a/benchmark/nlu_benchmark/services/LICENSE b/benchmark/nlu_benchmark/services/LICENSE deleted file mode 100644 index 94a9ed0..0000000 --- a/benchmark/nlu_benchmark/services/LICENSE +++ /dev/null @@ -1,674 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. diff --git a/benchmark/nlu_benchmark/services/Pipfile b/benchmark/nlu_benchmark/services/Pipfile deleted file mode 100644 index 9bd92f2..0000000 --- a/benchmark/nlu_benchmark/services/Pipfile +++ /dev/null @@ -1,13 +0,0 @@ -[[source]] -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" - -[packages] -requests = "*" - -[dev-packages] -pylint = "*" - -[requires] -python_version = "3.6" diff --git a/benchmark/nlu_benchmark/services/Pipfile.lock b/benchmark/nlu_benchmark/services/Pipfile.lock deleted file mode 100644 index 2297a09..0000000 --- a/benchmark/nlu_benchmark/services/Pipfile.lock +++ /dev/null @@ -1,135 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "5385c98d02bfcf11fb6a74ae72ea941086c6324557da061a89eea4194546523e" - }, - "pipfile-spec": 6, - "requires": { - "python_version": "3.6" - }, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "certifi": { - "hashes": [ - "sha256:13e698f54293db9f89122b0581843a782ad0934a4fe0172d2a980ba77fc61bb7", - "sha256:9fa520c1bacfb634fa7af20a76bcbd3d5fb390481724c597da32c719a7dca4b0" - ], - "version": "==2018.4.16" - }, - "chardet": { - "hashes": [ - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" - ], - "version": "==3.0.4" - }, - "idna": { - "hashes": [ - "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", - "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" - ], - "version": "==2.7" - }, - "requests": { - "hashes": [ - "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", - "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a" - ], - "index": "pypi", - "version": "==2.19.1" - }, - "urllib3": { - "hashes": [ - "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", - "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" - ], - "version": "==1.23" - } - }, - "develop": { - "astroid": { - "hashes": [ - "sha256:0ef2bf9f07c3150929b25e8e61b5198c27b0dca195e156f0e4d5bdd89185ca1a", - "sha256:fc9b582dba0366e63540982c3944a9230cbc6f303641c51483fa547dcc22393a" - ], - "version": "==1.6.5" - }, - "isort": { - "hashes": [ - "sha256:1153601da39a25b14ddc54955dbbacbb6b2d19135386699e2ad58517953b34af", - "sha256:b9c40e9750f3d77e6e4d441d8b0266cf555e7cdabdcff33c4fd06366ca761ef8", - "sha256:ec9ef8f4a9bc6f71eec99e1806bfa2de401650d996c59330782b89a5555c1497" - ], - "version": "==4.3.4" - }, - "lazy-object-proxy": { - "hashes": [ - "sha256:0ce34342b419bd8f018e6666bfef729aec3edf62345a53b537a4dcc115746a33", - "sha256:1b668120716eb7ee21d8a38815e5eb3bb8211117d9a90b0f8e21722c0758cc39", - "sha256:209615b0fe4624d79e50220ce3310ca1a9445fd8e6d3572a896e7f9146bbf019", - "sha256:27bf62cb2b1a2068d443ff7097ee33393f8483b570b475db8ebf7e1cba64f088", - "sha256:27ea6fd1c02dcc78172a82fc37fcc0992a94e4cecf53cb6d73f11749825bd98b", - "sha256:2c1b21b44ac9beb0fc848d3993924147ba45c4ebc24be19825e57aabbe74a99e", - "sha256:2df72ab12046a3496a92476020a1a0abf78b2a7db9ff4dc2036b8dd980203ae6", - "sha256:320ffd3de9699d3892048baee45ebfbbf9388a7d65d832d7e580243ade426d2b", - "sha256:50e3b9a464d5d08cc5227413db0d1c4707b6172e4d4d915c1c70e4de0bbff1f5", - "sha256:5276db7ff62bb7b52f77f1f51ed58850e315154249aceb42e7f4c611f0f847ff", - "sha256:61a6cf00dcb1a7f0c773ed4acc509cb636af2d6337a08f362413c76b2b47a8dd", - "sha256:6ae6c4cb59f199d8827c5a07546b2ab7e85d262acaccaacd49b62f53f7c456f7", - "sha256:7661d401d60d8bf15bb5da39e4dd72f5d764c5aff5a86ef52a042506e3e970ff", - "sha256:7bd527f36a605c914efca5d3d014170b2cb184723e423d26b1fb2fd9108e264d", - "sha256:7cb54db3535c8686ea12e9535eb087d32421184eacc6939ef15ef50f83a5e7e2", - "sha256:7f3a2d740291f7f2c111d86a1c4851b70fb000a6c8883a59660d95ad57b9df35", - "sha256:81304b7d8e9c824d058087dcb89144842c8e0dea6d281c031f59f0acf66963d4", - "sha256:933947e8b4fbe617a51528b09851685138b49d511af0b6c0da2539115d6d4514", - "sha256:94223d7f060301b3a8c09c9b3bc3294b56b2188e7d8179c762a1cda72c979252", - "sha256:ab3ca49afcb47058393b0122428358d2fbe0408cf99f1b58b295cfeb4ed39109", - "sha256:bd6292f565ca46dee4e737ebcc20742e3b5be2b01556dafe169f6c65d088875f", - "sha256:cb924aa3e4a3fb644d0c463cad5bc2572649a6a3f68a7f8e4fbe44aaa6d77e4c", - "sha256:d0fc7a286feac9077ec52a927fc9fe8fe2fabab95426722be4c953c9a8bede92", - "sha256:ddc34786490a6e4ec0a855d401034cbd1242ef186c20d79d2166d6a4bd449577", - "sha256:e34b155e36fa9da7e1b7c738ed7767fc9491a62ec6af70fe9da4a057759edc2d", - "sha256:e5b9e8f6bda48460b7b143c3821b21b452cb3a835e6bbd5dd33aa0c8d3f5137d", - "sha256:e81ebf6c5ee9684be8f2c87563880f93eedd56dd2b6146d8a725b50b7e5adb0f", - "sha256:eb91be369f945f10d3a49f5f9be8b3d0b93a4c2be8f8a5b83b0571b8123e0a7a", - "sha256:f460d1ceb0e4a5dcb2a652db0904224f367c9b3c1470d5a7683c0480e582468b" - ], - "version": "==1.3.1" - }, - "mccabe": { - "hashes": [ - "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", - "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" - ], - "version": "==0.6.1" - }, - "pylint": { - "hashes": [ - "sha256:a48070545c12430cfc4e865bf62f5ad367784765681b3db442d8230f0960aa3c", - "sha256:fff220bcb996b4f7e2b0f6812fd81507b72ca4d8c4d05daf2655c333800cb9b3" - ], - "index": "pypi", - "version": "==1.9.2" - }, - "six": { - "hashes": [ - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" - ], - "version": "==1.11.0" - }, - "wrapt": { - "hashes": [ - "sha256:d4d560d479f2c21e1b5443bbd15fe7ec4b37fe7e53d335d3b9b0a7b1226fe3c6" - ], - "version": "==1.10.11" - } - } -} diff --git a/benchmark/nlu_benchmark/services/__init__.py b/benchmark/nlu_benchmark/services/__init__.py index e69de29..7af7102 100644 --- a/benchmark/nlu_benchmark/services/__init__.py +++ b/benchmark/nlu_benchmark/services/__init__.py @@ -0,0 +1,4 @@ +import logging + + +logger = logging.getLogger('nlu_benchmark.services') diff --git a/benchmark/nlu_benchmark/services/app/__init__.py b/benchmark/v1/__init__.py similarity index 100% rename from benchmark/nlu_benchmark/services/app/__init__.py rename to benchmark/v1/__init__.py diff --git a/benchmark/nlu_benchmark/services/app/__main__.py b/benchmark/v1/__main__.py similarity index 100% rename from benchmark/nlu_benchmark/services/app/__main__.py rename to benchmark/v1/__main__.py diff --git a/benchmark/nlu_benchmark/services/app/bothub.py b/benchmark/v1/bothub.py similarity index 100% rename from benchmark/nlu_benchmark/services/app/bothub.py rename to benchmark/v1/bothub.py diff --git a/benchmark/nlu_benchmark/services/app/utils.py b/benchmark/v1/utils.py similarity index 100% rename from benchmark/nlu_benchmark/services/app/utils.py rename to benchmark/v1/utils.py diff --git a/benchmark/nlu_benchmark/services/app/wit.py b/benchmark/v1/wit.py similarity index 100% rename from benchmark/nlu_benchmark/services/app/wit.py rename to benchmark/v1/wit.py From 4a28b6cb2b7b5aad4796117389f7c86c64c51d2c Mon Sep 17 00:00:00 2001 From: periclesJ Date: Tue, 7 Aug 2018 17:48:22 -0300 Subject: [PATCH 10/23] Added decode from expressions.json from wit.ai --- benchmark/v1/__main__.py | 41 ++++++++++++++++++++++++++++++---------- benchmark/v1/bothub.py | 3 +-- benchmark/v1/wit.py | 17 +++++++++++++++++ 3 files changed, 49 insertions(+), 12 deletions(-) diff --git a/benchmark/v1/__main__.py b/benchmark/v1/__main__.py index 635d237..6bd7c90 100644 --- a/benchmark/v1/__main__.py +++ b/benchmark/v1/__main__.py @@ -2,7 +2,9 @@ import traceback from app import bothub +from app import wit from app.bothub import save_on_bothub, store_result, analyze_text, train, create_new_repository, delete_repository, write_csv_file +from app.wit import read_wit_and_push_bothub from app.utils import load_json_file#,percentage parser = argparse.ArgumentParser(description='Train and Test the accuracy from Bothub') @@ -14,25 +16,44 @@ def fill_bothub(args): expressions = load_json_file(args.filename) repository_data = create_new_repository(args) print('Uploading examples...') - for n, expression in enumerate(expressions['data']): - entities = [] - if (len(expression['entities']) > 0): - entities = expression['entities'] - try: - save_on_bothub(repository_data[0], expression['text'], entities, expression['intent']) - except KeyError: - traceback.print_exc() - print('Skipping expression {} due to an error'.format(expression)) - + if args.file_source != 'wit': + for n, expression in enumerate(expressions['data']): + entities = [] + if (len(expression['entities']) > 0): + entities = expression['entities'] + try: + save_on_bothub(repository_data[0], expression['text'], entities, expression['intent']) + except KeyError: + traceback.print_exc() + print('Skipping expression {} due to an error'.format(expression)) + else: + for expression in expressions['data']: + entities = [] + if (len(read_wit_and_push_bothub(expression)[1])): + entities = read_wit_and_push_bothub(expression)[1] + save_on_bothub(repository_data[0],read_wit_and_push_bothub(expression)[0],entities,read_wit_and_push_bothub(expression)[2]) + train_repository(args) print('Generating report...') predict(args) delete_repository(args.ownernick, repository_data[1]) +def decode_expressions_from_wit(args): + expressions = load_json_file(args.filename) + for expression in expressions['data']: + read_wit_and_push_bothub(expression) + + +task_decode_wit = sub_tasks.add_parser('decode_wit', help='Reads a exported json from Wit') +task_decode_wit.add_argument('--filename', help='expressions.json') +task_decode_wit.set_defaults(func=decode_expressions_from_wit) + + task_fill_bothub = sub_tasks.add_parser('fill_bothub', help='Insert data from a source to Bothub') task_fill_bothub.add_argument('--repository', help='repository uuid destination on bothub') task_fill_bothub.add_argument('--ownernick', help='Owner nickname') +task_fill_bothub.add_argument('--file_source', help='source from the file') task_fill_bothub.add_argument('--slug', help='Repository slug') task_fill_bothub.add_argument('--lang', help='language of the bot on Bothub') task_fill_bothub.add_argument('--filename', help='name of the file that contains the phrases to be trained') diff --git a/benchmark/v1/bothub.py b/benchmark/v1/bothub.py index 19fadde..1066f18 100644 --- a/benchmark/v1/bothub.py +++ b/benchmark/v1/bothub.py @@ -35,7 +35,6 @@ def delete_repository(owner_nickname, repository_slug): def save_on_bothub(args, text, entities, intent): data = {'repository': args, 'text': text, 'entities': entities, 'intent': intent} response = requests.post('{0}/api/example/new/'.format(BOTHUB_APP_URL), headers=user_token_header, json=json.loads(json.dumps(data))) - # print(response) def analyze_text(text, language, owner_nickname, repository_slug): @@ -68,7 +67,7 @@ def train(owner_nickname, repository_slug): def write_csv_file(data, csv_footer): csv_headers = "Phrases,Expected intents,Bothub predicts,Confidence,Result\n" - with open('output1.csv', "w") as csv_file: + with open('output.csv', "w") as csv_file: csv_file.write(csv_headers) for line in data: csv_file.write(line) diff --git a/benchmark/v1/wit.py b/benchmark/v1/wit.py index 992f7eb..08e0043 100644 --- a/benchmark/v1/wit.py +++ b/benchmark/v1/wit.py @@ -33,6 +33,23 @@ def get_intent_from_input(intent, file_item): return None +def read_wit_and_push_bothub(expression): + text = expression['text'] + wit_entities = expression['entities'] + filtered_entities = [] + intent = '' + for entity in wit_entities: + if entity['entity'] == 'intent': + intent = entity['value'].strip('\"') + else: + new_entity = {} + new_entity['entity'] = entity['entity'] + new_entity['start'] = entity['start'] + new_entity['end'] = entity['end'] + filtered_entities.append(new_entity) + return [text, filtered_entities, intent] + + def analyze(text, token): params = {'q':text} headers = {'Authorization':WIT_TOKEN} From c4a813ec72dc1d988a91e69f4c838db7f9911ebd Mon Sep 17 00:00:00 2001 From: periclesJ Date: Thu, 9 Aug 2018 15:31:48 -0300 Subject: [PATCH 11/23] updated scripts --- benchmark/v1/__main__.py | 35 ++++++++++++++++++++++++++++------- benchmark/v1/bothub.py | 32 ++++++++++---------------------- benchmark/v1/utils.py | 3 --- benchmark/v1/wit.py | 32 ++++++++++++++++++++++++++++---- 4 files changed, 66 insertions(+), 36 deletions(-) diff --git a/benchmark/v1/__main__.py b/benchmark/v1/__main__.py index 6bd7c90..c7657b6 100644 --- a/benchmark/v1/__main__.py +++ b/benchmark/v1/__main__.py @@ -4,7 +4,7 @@ from app import bothub from app import wit from app.bothub import save_on_bothub, store_result, analyze_text, train, create_new_repository, delete_repository, write_csv_file -from app.wit import read_wit_and_push_bothub +from app.wit import read_wit_and_push_bothub, analyze, store_wit_result, write_csv_file_from_wit from app.utils import load_json_file#,percentage parser = argparse.ArgumentParser(description='Train and Test the accuracy from Bothub') @@ -29,13 +29,14 @@ def fill_bothub(args): else: for expression in expressions['data']: entities = [] - if (len(read_wit_and_push_bothub(expression)[1])): + if (len(read_wit_and_push_bothub(expression)[1]) > 0): entities = read_wit_and_push_bothub(expression)[1] save_on_bothub(repository_data[0],read_wit_and_push_bothub(expression)[0],entities,read_wit_and_push_bothub(expression)[2]) train_repository(args) print('Generating report...') predict(args) + predict_on_wit(args) delete_repository(args.ownernick, repository_data[1]) @@ -79,15 +80,35 @@ def predict(args): except KeyError: traceback.print_exc() print('Skipping expression {} due to an error'.format(expression)) - # bothub_accuracy = percentage(sum_bothub_hits, count) - # bothub_confidence_avg = sum_bothub_confidence/count - write_csv_file(treated_predicts,'Analized phrases: {0}\nCorrect predictions: {1}\nWrong predictions: {2}'.format(phrases_count,sum_bothub_hits,sum_bothub_fails)) - # print('Final Accuracy: {}'.format(bothub_accuracy)) - # print('Average Confidence: {}%'.format(bothub_confidence_avg)) + write_csv_file(treated_predicts,'Analized phrases: {0}\nSuccess average: {1}%\nWrong predictions: {2}'.format(phrases_count,int((sum_bothub_hits/phrases_count)*100),sum_bothub_fails)) + +def predict_on_wit(args): + treated_predicts = [] + phrases_count = 0 + sum_wit_hits = 0 + sum_wit_fails = 0 + expressions = load_json_file(args.testfilename) + for expression in expressions['data']: + phrases_count += 1 + try: + wit_result = wit.analyze(expression['value']) + if 'intent' in wit_result['entities']: + if wit_result['entities']['intent'][0]['value'] == expression['intent']: + sum_wit_hits += 1 + else: + sum_wit_fails += 1 + else: + sum_wit_fails += 1 + treated_predicts.append(store_wit_result(expression, wit_result)) + except KeyError: + traceback.print_exc() + print('Skipping expression {} due to an error'.format(expression)) + write_csv_file_from_wit(treated_predicts,'Analized phrases: {0}\nSuccess average: {1}%\nWrong predictions: {2}'.format(phrases_count,int((sum_wit_hits/phrases_count)*100),sum_wit_fails)) def open_repository(args): create_new_repository(args) + task_predict = sub_tasks.add_parser('open_repository', help='Predict from Bothub and check accuracy') task_predict.set_defaults(func=open_repository) diff --git a/benchmark/v1/bothub.py b/benchmark/v1/bothub.py index 1066f18..ac122d7 100644 --- a/benchmark/v1/bothub.py +++ b/benchmark/v1/bothub.py @@ -6,12 +6,6 @@ user_token_header = {'Authorization': BOTHUB_USER_TOKEN} nlp_authorization_header = {'Authorization': BOTHUB_NLP_TOKEN} -valid_predict = [] -invalid_predict = [] -phrases_count = 0 -sum_bothub_hits = 0 -sum_bothub_fails = 0 - def create_new_repository(args): data = { 'description':'temp_', @@ -27,36 +21,30 @@ def create_new_repository(args): def delete_repository(owner_nickname, repository_slug): params = {'owner__nickname': owner_nickname, 'slug': repository_slug} - response = requests.delete('{0}/api/repository/{1}/{2}/'.format(BOTHUB_APP_URL, owner_nickname, repository_slug),headers=user_token_header,params=params) + requests.delete('{0}/api/repository/{1}/{2}/'.format(BOTHUB_APP_URL, owner_nickname, repository_slug),headers=user_token_header,params=params) print('Saved output.csv') print('Removed') def save_on_bothub(args, text, entities, intent): data = {'repository': args, 'text': text, 'entities': entities, 'intent': intent} - response = requests.post('{0}/api/example/new/'.format(BOTHUB_APP_URL), headers=user_token_header, json=json.loads(json.dumps(data))) + requests.post('{0}/api/example/new/'.format(BOTHUB_APP_URL), headers=user_token_header, json=json.loads(json.dumps(data))) def analyze_text(text, language, owner_nickname, repository_slug): data = {'text': text, 'language':language} response = requests.post('{0}/api/repository/{1}/{2}/analyze/'.format(BOTHUB_APP_URL, owner_nickname, repository_slug), headers=user_token_header, data=data) - response_json = json.loads(response.content) - return response_json + return json.loads(response.content) def store_result(expression, bothub_result): - global phrases_count - global sum_bothub_hits - global sum_bothub_fails - phrases_count += 1 + detected_entities = '-' + for entity in bothub_result['answer']['entities']: + detected_entities += '{}-'.format(entity['entity']) if bothub_result['answer']['intent']['name'] == expression['intent']: - sum_bothub_hits += 1 - return '{0},{1},{2},{3}%,OK'.format(expression['value'], expression['intent'], bothub_result['answer']['intent']['name'],int(float(bothub_result['answer']['intent']['confidence'])*100)) + return '{0},{1},{2},{3}%,OK,{4}'.format(expression['value'], expression['intent'], bothub_result['answer']['intent']['name'],int(float(bothub_result['answer']['intent']['confidence'])*100),detected_entities) else: - sum_bothub_fails += 1 - return '{0},{1},{2},{3}%,FAILURE'.format(expression['value'], expression['intent'], bothub_result['answer']['intent']['name'],int(float(bothub_result['answer']['intent']['confidence'])*100)) - #valid_predict.extend(invalid_predict) - #write_csv_file(valid_predict,'Analized phrases: {0}\nCorrect predictions: {1}\nWrong predictions: {2}'.format(phrases_count,sum_bothub_hits,sum_bothub_fails)) + return '{0},{1},{2},{3}%,FAILURE,{4}'.format(expression['value'], expression['intent'], bothub_result['answer']['intent']['name'],int(float(bothub_result['answer']['intent']['confidence'])*100),detected_entities) def train(owner_nickname, repository_slug): @@ -66,8 +54,8 @@ def train(owner_nickname, repository_slug): def write_csv_file(data, csv_footer): - csv_headers = "Phrases,Expected intents,Bothub predicts,Confidence,Result\n" - with open('output.csv', "w") as csv_file: + csv_headers = "Phrases,Expected intents,Bothub predicts,Confidence,Result,Detected entities\n" + with open('Bothub_output.csv', "w") as csv_file: csv_file.write(csv_headers) for line in data: csv_file.write(line) diff --git a/benchmark/v1/utils.py b/benchmark/v1/utils.py index c8c88bd..51c8c4b 100644 --- a/benchmark/v1/utils.py +++ b/benchmark/v1/utils.py @@ -3,9 +3,6 @@ def load_json_file(entity_file): - # with open(entity_file) as json_file: - # data = json.dumps(json_file.read()) - # data_dict = json.loads(data) data_dict = json.load(io.open(entity_file, 'r', encoding='utf-8-sig')) return data_dict diff --git a/benchmark/v1/wit.py b/benchmark/v1/wit.py index 08e0043..dfbf429 100644 --- a/benchmark/v1/wit.py +++ b/benchmark/v1/wit.py @@ -1,9 +1,9 @@ import os +import json import requests from app.utils import load_json_file -from app.settings import WIT_APP_URL, WIT_TOKEN - +from app.settings import WIT_APP_URL, WIT_TOKEN, WIT_API_VERSION def get_expressions_data(source): files = os.listdir(source) @@ -50,9 +50,33 @@ def read_wit_and_push_bothub(expression): return [text, filtered_entities, intent] -def analyze(text, token): - params = {'q':text} +def analyze(text): + params = {'q':text, 'v':WIT_API_VERSION} headers = {'Authorization':WIT_TOKEN} response = requests.get(WIT_APP_URL, params=params, headers=headers) response_json = json.loads(response.content) return response_json + +def store_wit_result(expression, wit_result): + prediction_result = 'OK' + detected_entities = '-' + if 'intent' in wit_result['entities']: + if wit_result['entities']['intent'][0]['value'] != expression['intent']: + prediction_result = 'FAILURE' + for key in wit_result['entities']: + if key != 'intent': + detected_entities += '{}-'.format(key) + return '{0},{1},{2},{3}%,{4},{5}'.format(expression['value'], expression['intent'], wit_result['entities']['intent'][0]['value'],int(float(wit_result['entities']['intent'][0]['confidence'])*100),prediction_result,detected_entities) + else: + prediction_result = 'FAILURE' + return '{0},{1},,,{2},'.format(expression['value'], expression['intent'],prediction_result) + + +def write_csv_file_from_wit(data, csv_footer): + csv_headers = "Phrases,Expected intents,wit predicts,Confidence,Result,Detected entities\n" + with open('wit_output.csv', "w") as csv_file: + csv_file.write(csv_headers) + for line in data: + csv_file.write(line) + csv_file.write('\n') + csv_file.write('\n' + csv_footer) \ No newline at end of file From 295c411a08da31eb401e20ad98f3b661c13be1a0 Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Fri, 10 Aug 2018 16:24:58 -0300 Subject: [PATCH 12/23] Add train examples in bothub --- benchmark/Pipfile | 1 + benchmark/Pipfile.lock | 10 ++++- benchmark/nlu_benchmark/cli/run.py | 47 ++++++++++++++++++---- benchmark/nlu_benchmark/services/bothub.py | 18 ++++++++- 4 files changed, 67 insertions(+), 9 deletions(-) diff --git a/benchmark/Pipfile b/benchmark/Pipfile index 04ba665..74409b4 100644 --- a/benchmark/Pipfile +++ b/benchmark/Pipfile @@ -7,6 +7,7 @@ name = "pypi" plac = "*" requests = "*" pyyaml = "*" +tqdm = "*" [dev-packages] "flake8" = "*" diff --git a/benchmark/Pipfile.lock b/benchmark/Pipfile.lock index 4af1286..79ac533 100644 --- a/benchmark/Pipfile.lock +++ b/benchmark/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "2ebcdfec05ef7338ee9e78ac18c638e162baf944079a26f3fe5894aa2e2243e7" + "sha256": "492c6755f86d73aa1fbdd359074ccb766d2f22476e8b693c5fb4903cddb72698" }, "pipfile-spec": 6, "requires": { @@ -70,6 +70,14 @@ "index": "pypi", "version": "==2.19.1" }, + "tqdm": { + "hashes": [ + "sha256:536e5a0205b6401d8aaf04b21469949c178dbe3642e3c76d3bb494a83cb22a10", + "sha256:60bbaa6700e87a250f6abcbbd7ddb33243ad592240ba46afce5305b15b406fad" + ], + "index": "pypi", + "version": "==4.24.0" + }, "urllib3": { "hashes": [ "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", diff --git a/benchmark/nlu_benchmark/cli/run.py b/benchmark/nlu_benchmark/cli/run.py index 19c3218..c2702b6 100644 --- a/benchmark/nlu_benchmark/cli/run.py +++ b/benchmark/nlu_benchmark/cli/run.py @@ -3,6 +3,8 @@ import json import requests +from tqdm import tqdm + from . import logger from ..services.bothub import Bothub @@ -37,6 +39,8 @@ def run(data_file_path, data_file_type='yaml', bothub_user_token=None): logger.info('Starting Bothub benchmark...') bothub = Bothub(user_token=bothub_user_token) + ## Create a temporary repository + create_repository_response = bothub.create_temporary_repository( data.get('language')) @@ -50,14 +54,43 @@ def run(data_file_path, data_file_type='yaml', bothub_user_token=None): temporary_repository = create_repository_response.json() logger.info(f'Bothub temporary repository created "{temporary_repository.get("name")}"') - delete_repository_response = bothub.delete_repository( - temporary_repository.get('owner__nickname'), - temporary_repository.get('slug')) try: - delete_repository_response.raise_for_status() + ## Train + + train = data.get('train') + total_train = len(train) + logger.info(f'{total_train} examples to train in Bothub') + with tqdm(total=total_train, unit='examples') as pbar: + for example in train: + example_submit_response = bothub.submit_example( + temporary_repository.get('uuid'), + example) + example_submit_response.raise_for_status() + logger.debug(f'example trained {example_submit_response.json()}') + pbar.update(1) + logger.info('Examples submitted!') + + logger.info('Training...') + train_response = bothub.train( + temporary_repository.get('owner__nickname'), + temporary_repository.get('slug')) + train_response.raise_for_status() + logger.debug(f'repository train response {train_response.json()}') + logger.info('Repository trained') except Exception as e: - logger.warning('Bothub temporary repository not deleted, manually delete') - logger.debug(delete_repository_response.text) + raise e + finally: + ## Delete a temporary repository + + delete_repository_response = bothub.delete_repository( + temporary_repository.get('owner__nickname'), + temporary_repository.get('slug')) + + try: + delete_repository_response.raise_for_status() + except Exception as e: + logger.warning('Bothub temporary repository not deleted, manually delete') + logger.debug(delete_repository_response.text) - logger.info(f'Bothub temporary repository deleted') + logger.info(f'Bothub temporary repository deleted') diff --git a/benchmark/nlu_benchmark/services/bothub.py b/benchmark/nlu_benchmark/services/bothub.py index 8c56d39..e5f0e4a 100644 --- a/benchmark/nlu_benchmark/services/bothub.py +++ b/benchmark/nlu_benchmark/services/bothub.py @@ -101,4 +101,20 @@ def create_temporary_repository(self, language): def delete_repository(self, owner_nickname, repository_slug): return Bothub.api_request( f'repository/{owner_nickname}/{repository_slug}', - replace_method='DELETE') + replace_method='DELETE', + user_token=self.user_token) + + def submit_example(self, repository_uuid, example, language=None): + data = example + data.update({'repository': repository_uuid}) + if language: + data.update({'language': language}) + return Bothub.api_request( + 'example/new', + data, + user_token=self.user_token) + + def train(self, owner_nickname, slug): + return Bothub.api_request( + f'repository/{owner_nickname}/{slug}/train', + user_token=self.user_token) From 8d77905ed22a7c3b048b180e4297aa499754613f Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Fri, 10 Aug 2018 16:56:19 -0300 Subject: [PATCH 13/23] Add type test --- benchmark/datafiles/binary.yaml | 8 +++++ benchmark/nlu_benchmark/cli/run.py | 40 +++++++++++++++++++--- benchmark/nlu_benchmark/services/bothub.py | 10 ++++++ 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/benchmark/datafiles/binary.yaml b/benchmark/datafiles/binary.yaml index ec70fae..b4f59e5 100644 --- a/benchmark/datafiles/binary.yaml +++ b/benchmark/datafiles/binary.yaml @@ -6,9 +6,17 @@ train: intent: "affirmative" - text: "okay" intent: "affirmative" + - text: "of course" + intent: "affirmative" - text: "no" intent: "negative" - text: "nop" intent: "negative" - text: "not" intent: "negative" + - text: "maybe" + intent: "maybe" + - text: "maybe yes" + intent: "maybe" + - text: "maybe no" + intent: "maybe" diff --git a/benchmark/nlu_benchmark/cli/run.py b/benchmark/nlu_benchmark/cli/run.py index c2702b6..8a3a2f0 100644 --- a/benchmark/nlu_benchmark/cli/run.py +++ b/benchmark/nlu_benchmark/cli/run.py @@ -12,8 +12,10 @@ @plac.annotations( data_file_path=plac.Annotation(), data_file_type=plac.Annotation(kind='option', choices=['yaml', 'json']), - bothub_user_token=plac.Annotation(kind='option')) -def run(data_file_path, data_file_type='yaml', bothub_user_token=None): + bothub_user_token=plac.Annotation(kind='option'), + type_tests=plac.Annotation(kind='flag', abbrev='T')) +def run(data_file_path, data_file_type='yaml', bothub_user_token=None, + type_tests=False): try: data_file = open(data_file_path, 'r') except Exception as e: @@ -65,9 +67,10 @@ def run(data_file_path, data_file_type='yaml', bothub_user_token=None): for example in train: example_submit_response = bothub.submit_example( temporary_repository.get('uuid'), - example) + example, + data.get('language')) + logger.debug(f'example trained {example_submit_response.text}') example_submit_response.raise_for_status() - logger.debug(f'example trained {example_submit_response.json()}') pbar.update(1) logger.info('Examples submitted!') @@ -75,9 +78,36 @@ def run(data_file_path, data_file_type='yaml', bothub_user_token=None): train_response = bothub.train( temporary_repository.get('owner__nickname'), temporary_repository.get('slug')) + logger.debug(f'repository train response {train_response.text}') train_response.raise_for_status() - logger.debug(f'repository train response {train_response.json()}') logger.info('Repository trained') + + ## Test + if type_tests: + logger.warning('Typing mode ON, press CTRL + C to exit') + try: + while True: + text = input('Type a text to test: ') + analyze_response = bothub.analyze( + temporary_repository.get('owner__nickname'), + temporary_repository.get('slug'), + text, + data.get('language')) + + logger.debug(f'analyze response {analyze_response.text}') + analyze_response.raise_for_status() + analyze_json = analyze_response.json() + analyze_answer = analyze_json.get('answer') + analyze_intent = analyze_answer.get('intent') + logger.info('Bothub return:') + logger.info(f' - intent: {analyze_intent.get("name", "[not detected]")} ' + + f'({analyze_intent.get("confidence", 0) * 100}%)') + except KeyboardInterrupt as e: + logger.info('Tests finished!') + else: + pass # TODO: load from data file + except requests.exceptions.HTTPError as e: + raise e except Exception as e: raise e finally: diff --git a/benchmark/nlu_benchmark/services/bothub.py b/benchmark/nlu_benchmark/services/bothub.py index e5f0e4a..3700d9a 100644 --- a/benchmark/nlu_benchmark/services/bothub.py +++ b/benchmark/nlu_benchmark/services/bothub.py @@ -118,3 +118,13 @@ def train(self, owner_nickname, slug): return Bothub.api_request( f'repository/{owner_nickname}/{slug}/train', user_token=self.user_token) + + def analyze(self, owner_nickname, slug, text, language): + data = { + 'text': text, + 'language': language, + } + return Bothub.api_request( + f'repository/{owner_nickname}/{slug}/analyze', + data, + user_token=self.user_token) From f010fd62dec586a383696b13ef1cd7f8c5bd42d2 Mon Sep 17 00:00:00 2001 From: periclesJ Date: Mon, 13 Aug 2018 15:39:09 -0300 Subject: [PATCH 14/23] Updated report generator --- benchmark/v1/__main__.py | 7 +++---- benchmark/v1/bothub.py | 30 ++++++++++++++++++++++++++---- benchmark/v1/wit.py | 38 +++++++++++++++++++++++++++++++------- 3 files changed, 60 insertions(+), 15 deletions(-) diff --git a/benchmark/v1/__main__.py b/benchmark/v1/__main__.py index c7657b6..d40fdd7 100644 --- a/benchmark/v1/__main__.py +++ b/benchmark/v1/__main__.py @@ -5,8 +5,7 @@ from app import wit from app.bothub import save_on_bothub, store_result, analyze_text, train, create_new_repository, delete_repository, write_csv_file from app.wit import read_wit_and_push_bothub, analyze, store_wit_result, write_csv_file_from_wit -from app.utils import load_json_file#,percentage - +from app.utils import load_json_file parser = argparse.ArgumentParser(description='Train and Test the accuracy from Bothub') sub_tasks = parser.add_subparsers(title='Tasks') @@ -71,7 +70,7 @@ def predict(args): for expression in expressions['data']: phrases_count += 1 try: - bothub_result = bothub.analyze_text(expression['value'], args.lang, args.ownernick, args.slug) + bothub_result = bothub.analyze_text(expression['text'], args.lang, args.ownernick, args.slug) treated_predicts.append(store_result(expression,bothub_result)) if bothub_result['answer']['intent']['name'] == expression['intent']: sum_bothub_hits += 1 @@ -92,7 +91,7 @@ def predict_on_wit(args): for expression in expressions['data']: phrases_count += 1 try: - wit_result = wit.analyze(expression['value']) + wit_result = wit.analyze(expression['text']) if 'intent' in wit_result['entities']: if wit_result['entities']['intent'][0]['value'] == expression['intent']: sum_wit_hits += 1 diff --git a/benchmark/v1/bothub.py b/benchmark/v1/bothub.py index ac122d7..a9eca4a 100644 --- a/benchmark/v1/bothub.py +++ b/benchmark/v1/bothub.py @@ -39,12 +39,34 @@ def analyze_text(text, language, owner_nickname, repository_slug): def store_result(expression, bothub_result): detected_entities = '-' + known_entities = '-' + bothub_entities_count = 0 + known_entities_count = 0 + matched_entities = 0 + entity_result = '' + entities_percentage = 0 + for each_known_entity in expression['entities']: + known_entities_count += 1 + known_entities += '{0}("{1}")-'.format(each_known_entity['entity'],each_known_entity['value']) for entity in bothub_result['answer']['entities']: - detected_entities += '{}-'.format(entity['entity']) + bothub_entities_count += 1 + detected_entities += '{0}("{1}")-'.format(entity['entity'],entity['value']) + for each_known_entity in expression['entities']: + if entity['entity'] == each_known_entity['entity']: + matched_entities += 1 + if bothub_entities_count == known_entities_count == matched_entities: + entity_result = 'OK' + entities_percentage = 100 + elif bothub_entities_count != known_entities_count and matched_entities > 0: + entities_percentage = (bothub_entities_count/known_entities_count)*100 + entity_result = 'PARCIAL' + else: + entities_percentage = 0 + entity_result = 'FAILURE' if bothub_result['answer']['intent']['name'] == expression['intent']: - return '{0},{1},{2},{3}%,OK,{4}'.format(expression['value'], expression['intent'], bothub_result['answer']['intent']['name'],int(float(bothub_result['answer']['intent']['confidence'])*100),detected_entities) + return '{0},{1},{2},{3}%,OK,{4},{5},{6},{7}%'.format(expression['text'], expression['intent'], bothub_result['answer']['intent']['name'],int(float(bothub_result['answer']['intent']['confidence'])*100),detected_entities,known_entities,entity_result,entities_percentage) else: - return '{0},{1},{2},{3}%,FAILURE,{4}'.format(expression['value'], expression['intent'], bothub_result['answer']['intent']['name'],int(float(bothub_result['answer']['intent']['confidence'])*100),detected_entities) + return '{0},{1},{2},{3}%,FAILURE,{4},{5},{6},{7}%'.format(expression['text'], expression['intent'], bothub_result['answer']['intent']['name'],int(float(bothub_result['answer']['intent']['confidence'])*100),detected_entities,known_entities,entity_result,entities_percentage) def train(owner_nickname, repository_slug): @@ -54,7 +76,7 @@ def train(owner_nickname, repository_slug): def write_csv_file(data, csv_footer): - csv_headers = "Phrases,Expected intents,Bothub predicts,Confidence,Result,Detected entities\n" + csv_headers = "Phrases,Expected intents,Bothub predicts,Confidence accuracy,Result by Intent,Detected entities,Known entities,Result by Entity, Entity accuracy\n" with open('Bothub_output.csv', "w") as csv_file: csv_file.write(csv_headers) for line in data: diff --git a/benchmark/v1/wit.py b/benchmark/v1/wit.py index dfbf429..8b1714e 100644 --- a/benchmark/v1/wit.py +++ b/benchmark/v1/wit.py @@ -60,20 +60,44 @@ def analyze(text): def store_wit_result(expression, wit_result): prediction_result = 'OK' detected_entities = '-' + known_entities = '-' + wit_entities_count = 0 + known_entities_count = 0 + matched_entities = 0 + entity_result = '' + entities_percentage = 0 + for known_entity in expression['entities']: + known_entities_count += 1 + known_entities += '{0}("{1}")-'.format(known_entity['entity'],known_entity['value']) + + for key in wit_result['entities']: + if key != 'intent': + wit_entities_count += 1 + detected_entities += '{0}("{1}")-'.format(key,(wit_result['entities']).get(key)[0]['value']) + for entity in expression['entities']: + if key == entity['entity']: + matched_entities += 1 + if wit_entities_count == known_entities_count == matched_entities: + entity_result = 'OK' + entities_percentage = 100 + elif wit_entities_count != known_entities_count and matched_entities > 0: + entities_percentage = (wit_entities_count/known_entities_count)*100 + entity_result = 'PARCIAL' + else: + entities_percentage = 0 + entity_result = 'FAILURE' + if 'intent' in wit_result['entities']: if wit_result['entities']['intent'][0]['value'] != expression['intent']: - prediction_result = 'FAILURE' - for key in wit_result['entities']: - if key != 'intent': - detected_entities += '{}-'.format(key) - return '{0},{1},{2},{3}%,{4},{5}'.format(expression['value'], expression['intent'], wit_result['entities']['intent'][0]['value'],int(float(wit_result['entities']['intent'][0]['confidence'])*100),prediction_result,detected_entities) + prediction_result = 'FAILURE' + return '{0},{1},{2},{3}%,{4},{5},{6},{7}%'.format(expression['text'], expression['intent'], wit_result['entities']['intent'][0]['value'],int(float(wit_result['entities']['intent'][0]['confidence'])*100),prediction_result,detected_entities,known_entities,entity_result,entities_percentage) else: prediction_result = 'FAILURE' - return '{0},{1},,,{2},'.format(expression['value'], expression['intent'],prediction_result) + return '{0},{1},,,{2},,{3},{4},{5}%'.format(expression['text'], expression['intent'],prediction_result,known_entities, entity_result,entities_percentage) def write_csv_file_from_wit(data, csv_footer): - csv_headers = "Phrases,Expected intents,wit predicts,Confidence,Result,Detected entities\n" + csv_headers = "Phrases,Expected intents,wit predicts,Confidence accuracy,Result,Detected entities,Known entities,Result by Entity, Entity accuracy\n" with open('wit_output.csv', "w") as csv_file: csv_file.write(csv_headers) for line in data: From cf32c4d0f6c4c143be59e04bee59ee74b5f07a41 Mon Sep 17 00:00:00 2001 From: periclesJ Date: Tue, 14 Aug 2018 11:32:37 -0300 Subject: [PATCH 15/23] small adjustment to avoid crashing --- benchmark/v1/__main__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/benchmark/v1/__main__.py b/benchmark/v1/__main__.py index d40fdd7..0846184 100644 --- a/benchmark/v1/__main__.py +++ b/benchmark/v1/__main__.py @@ -28,9 +28,10 @@ def fill_bothub(args): else: for expression in expressions['data']: entities = [] - if (len(read_wit_and_push_bothub(expression)[1]) > 0): - entities = read_wit_and_push_bothub(expression)[1] - save_on_bothub(repository_data[0],read_wit_and_push_bothub(expression)[0],entities,read_wit_and_push_bothub(expression)[2]) + if 'entities' in expression: + if (len(read_wit_and_push_bothub(expression)[1]) > 0): + entities = read_wit_and_push_bothub(expression)[1] + save_on_bothub(repository_data[0],read_wit_and_push_bothub(expression)[0],entities,read_wit_and_push_bothub(expression)[2]) train_repository(args) print('Generating report...') From aa8421fb54324168145ba26625e764aa268d3dc9 Mon Sep 17 00:00:00 2001 From: Douglas Paz Date: Wed, 22 Aug 2018 08:22:15 -0300 Subject: [PATCH 16/23] Init test in run --- benchmark/datafiles/binary.yaml | 10 +++++++ benchmark/nlu_benchmark/cli/run.py | 44 ++++++++++++++++++++---------- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/benchmark/datafiles/binary.yaml b/benchmark/datafiles/binary.yaml index b4f59e5..de8ecb1 100644 --- a/benchmark/datafiles/binary.yaml +++ b/benchmark/datafiles/binary.yaml @@ -20,3 +20,13 @@ train: intent: "maybe" - text: "maybe no" intent: "maybe" +tests: + - text: "yes" + expected: + - intent: "affirmative" + - text: "no" + expected: + - intent: "negative" + - text: "maybe" + expected: + - intent: "maybe" diff --git a/benchmark/nlu_benchmark/cli/run.py b/benchmark/nlu_benchmark/cli/run.py index 8a3a2f0..474d3f5 100644 --- a/benchmark/nlu_benchmark/cli/run.py +++ b/benchmark/nlu_benchmark/cli/run.py @@ -83,29 +83,43 @@ def run(data_file_path, data_file_type='yaml', bothub_user_token=None, logger.info('Repository trained') ## Test + + test_result = [] + + def analyze_wrapper(text, expected={}): + analyze_response = bothub.analyze( + temporary_repository.get('owner__nickname'), + temporary_repository.get('slug'), + text, + data.get('language')) + logger.debug(f'analyze response {analyze_response.text}') + analyze_response.raise_for_status() + analyze = analyze_response.json() + analyze_answer = analyze.get('answer') + analyze_intent = analyze_answer.get('intent') + logger.info('Bothub return:') + logger.info(f' - intent: {analyze_intent.get("name", "[not detected]")} ' + + f'({analyze_intent.get("confidence", 0) * 100}%)') + test_result.append({ + 'text': text, + 'intent': analyze_intent, + 'expected': expected, + 'response': analyze, + }) + if type_tests: logger.warning('Typing mode ON, press CTRL + C to exit') try: while True: text = input('Type a text to test: ') - analyze_response = bothub.analyze( - temporary_repository.get('owner__nickname'), - temporary_repository.get('slug'), - text, - data.get('language')) - - logger.debug(f'analyze response {analyze_response.text}') - analyze_response.raise_for_status() - analyze_json = analyze_response.json() - analyze_answer = analyze_json.get('answer') - analyze_intent = analyze_answer.get('intent') - logger.info('Bothub return:') - logger.info(f' - intent: {analyze_intent.get("name", "[not detected]")} ' + - f'({analyze_intent.get("confidence", 0) * 100}%)') + analyze_wrapper(text) except KeyboardInterrupt as e: logger.info('Tests finished!') else: - pass # TODO: load from data file + for test in data.get('tests'): + analyze_wrapper(test.get('text'), test.get('expected')) + + logger.debug(f'test_result: {test_result}') except requests.exceptions.HTTPError as e: raise e except Exception as e: From 91f7cd596270318fc112b68f8454fa3955b9b728 Mon Sep 17 00:00:00 2001 From: periclesJ Date: Wed, 22 Aug 2018 11:09:54 -0300 Subject: [PATCH 17/23] Added bothub training from wit zipfile --- benchmark/datafiles/Benchmark_Training.zip | Bin 0 -> 16758 bytes benchmark/datafiles/train_config.yaml | 7 +++ benchmark/nlu_benchmark/cli/run.py | 54 +++++++++++++-------- benchmark/nlu_benchmark/services/bothub.py | 16 +++--- 4 files changed, 52 insertions(+), 25 deletions(-) create mode 100644 benchmark/datafiles/Benchmark_Training.zip create mode 100644 benchmark/datafiles/train_config.yaml diff --git a/benchmark/datafiles/Benchmark_Training.zip b/benchmark/datafiles/Benchmark_Training.zip new file mode 100644 index 0000000000000000000000000000000000000000..0d4075f2b749ed2beda46019306bb2565d993574 GIT binary patch literal 16758 zcmaib1yq&I_ch(!ARyf!Al;yJcQ*)^ZV(hCrMtTu1WD;ex*O>ZQ3S4%-^JJ8EAqX6 z92aZdyTHAk+0U6XXU^WEAPotH1qKHP2j)nHBMWwCJOX_dF|{!^w>EUJ)K_sZ1lRy< z%$^w9*)dr-+S)ir@S=1uBLx7{JNg6P)RN%gaYOR5mWe`kRYre7Q$3Qy7z<*(ezNbV z4^{}v{f?1-w!OHxZYp1IB<5!Ny?H6iN9+FR0Dk)ke5aZ9_N*5}+nd$8*&mPB80Ag7 zgkfb{Qm@`Mbw$bR2)L?<@JC(%gj?*jI)K9LTVE!#d?)H5Kk!zj zU)DDoHbuc;s5k`}eFsyJhJ_QXG1txj&4K|O4D|i~nkVId&(qY#3E%`Ub$sFkur~cU zW9tZ(+Zl@j)5(ryp0x2ND!?Yxf3%P&rtr@$E-+jnX~vV3pi^a@^E|6--Ijuj+(GYB z9L$2tW6ggIi`&N?)y5;O;*rcph_JW&gISG7BJ+pCs9R1U)Cg&xLRJeE7Z(4h37}JGSxJ7PS5_j*o42F2{H1M1?yJklNU3;-9-@})3dv@nj`Q$~(Cz#|^#zw(B z$*t9mkJHgS(-hEvk!#J=I&%6E_lwG4-gwkoCYTm{k?KJ6ba;>ns=qK{w|yJylQGqavSjD}kF4NE6iU zIHh?ZurlIFZj`lA*2m|=5&hCi+0$@qR4ZsFJkqdyuPgB0#Z)x!%?pPZD=m1UvVN$6 zu9LS&Kr|=h8nwZCR6$G~H~9KHTy1Z+!|=pY`TW70H`KiEo{;hg9|e6POWUFXGFTVX z4V*ve!PNNDpE)1Zi6dp$Y8*F71YlV z6SS!ZqbQeWu}V|MoPJ+BkQhmtmTJM0R-j9RWn=Few{u4L72{@DS=d#OEbTV6f~19a z)859uV`lC5o%k#%Bizos%#E_rC+kT-2qvsL--VC-*7ncvz| zXk9O7(yd51^-)SHdo~~4z8RN6j8!qvTPcNkC`PIO@@4=wPM~=G`D)^#S!_CQqcl}y z^gu7ua;7{f7^zhB(Q^}}$U>5`s77@)LRgq!ku)d)&#%uWGf;apU=gO9l0RVOosex4 zj*AG;(aMlV&TFMLG=>^>h#CiBYaRykf6=g?Mbi^4(q91+Se5F#a=3sHSRZ!R8xgP4 z^C@NRgQlIUdA)x<4(#IcK1}Ct3J9c)`Z8R0wzT>v=hMfcArUHvAsk!RQ$c=a=%8S0 zTjD@(k6aq-^U6!Jbsq6pmJXp&D%8R-Utc#mML=o_N#i!F%c_&ow-*uc>mi-*5uNfk zLQnS5Yer=Bg6cGg0zW_u*Fii5Yzz!U~lOPI8MrOD02slH72To4|u(V_YhybE;8e7SqIV!PT zE*NF!D?Pz)DY!B|JSYWCQ18LDP5&1PjBP>r&IS}j|KvL72!51K5+pHTD#S}0rqO2c z9Tn5e_96uuvCHbP?8W94kzAy-wvks`FAx2|D086faV(h}0D)T@BBY#DNwUP15#NLC zEX;d0W%Zy9m@m0Beso3qMOjVaSa%VClOOtOm&<}*Doh%C$FPCZPwkEPC9uJx8d>sd z$M_~}^!0s;y|E{LHfn+j_&hbkOR#6ndyp5%zDh6(L?~d$f9^724{!nI%Hk~5?e>Qr zBHq#<7@A014w|IVgETPwg$5^povpsHtu?^J@SoKbBWvT!j3x%;_TO}N@LtL(p)00x zqfripoqOi#=a875hfB{D;8U#P27TRQo9>_FSrE%Xk~tN!_qMyDdeC_$faPUKUt}Ed zR}weu=8S9tTii3M9X!IvA6!OPBrLMxml16~3K>;H1JNxHC&kv|{DWB-+F@j9lgt>0 z4S02k%H&6i@cJ-0iv7_HsA9@a@gOR-YXWO1S+{#nT$XRl=-YS~_g-2Jv#aP-K+oAE zdFv>f^}Y6b3E?k%X=g0q`Az(lh*^4jX5Q2D1-1Pq!3kwjjUtsktJ>mtiTuMVN9Xu` z+>dPR297t^l*co*$#$LgdZ(5`Ud$TN&vHiL`4MvSzmdOAWr6YGMLFo&I;`Z+oBqT#o2J7zr(ukS_%_CuFB9d6)7lRjwVxBV z+hZ}#ph8^u{5xuN$@Di;4KTTR#pccc=@3}Tud<$-4qYaBKe#$aOX?lEHqV(Er+CWu zs$hJlqt$|fGV-#yXd@P)Z=_1iUpELs=4?HEva4*qjF!rnEMo8s0lIUj?b9>PDT(o@ zV9^(iy~hx@F_;Ej$J`B?`22%0__u7bH@t_=nfI5yW36fTgLS=o3^JxQllrRx$ij3s19H6MU;FLYqXuX{|1<;GN05o%*2WS4b)n{#MVrpfp zZ*A-JPhdDl+@@l*=SNov@6A$9HrjJMeW@q#V#`SLtRS_mCr*|b;bh&vKwSjjzt|Rm zQ;4Pvj7gj^Btll?JrRi1rl7YZH0tAGC1ka*KaW%$G{)voeVBBnFHF%oX8QKgS>EbeY>t4Rd~5N~wNE1|@Lb~JY zA;)A;W}AVHTtdAdhk6mcfsrdq0y0SsVY?%xFD1z% z%$l*h%ryDZFxq4A@3-G5Bu_xs!;Ya&~s<-ItU(VAJxqR}ZN#g6FzU61Cm&>IW z#199=LmA|6HIJbUz}nF2pDZ0Ee9H}}=E+W~`N<8F7*Ax+Jcl)i8#dzZ5O`x!j4*H2 z1x~N)j28_uaFKZEU3yNP#%>mtPihd!xmocwc7u*3Wc@z*F-0S=(p z_JVo(gma4U8vDQtD;2TH85CiqRQeu&;+$k>gxMSC#{N(wbTnpIZI~YIm|B1q%Z>o^|XOAR=-Zakon-Gh@CuZ^Fqq@tDZIFsHzOV@Zgy-lY& zIO}p7E4cd+9DlEFp8SmiLkA~QeUsa*-NEjk+!7Y4h~Nu4zb^BO_EqT_0B0xAf;hsB zW+b#ZMfpZh-iOGRV4IM5SbU@Oy8aRoghnz(9vFhRZN$XxI8kFmS|{{EO*?x~`G_>E zFl)72jR%fIpBwMhXtp0tS%HXyVx{%V7;iC0{RMkWg%Lef#8O4x`NC*w?es!k9zre~ zTPqlQf6CWO^%;bNTGGR`9I!8Uy1%AvF4#(q=uqIx-?*1{wxYa6qI}D>viI>tw1ufe zux8;U8YT^IUNftQMc;el^PgJ|xxCe4CusJ=59XS`#)zSlt(C1A;8)QTD{BKuisb+6 zbO+oi?(cvnHt`38o}ePw{EVO2Dxpb*q%oNR}8vet`B{J-^2APCg7AOrj0}LuCujoM8cF3lRKgZarQb58^Kboik30pm>9;X?MWsT5g%FW{ z9e!iS&N`a+$%%>iZ;sd{Q!0406Ve#Z4e5s zxz8el2(Evy^Dum4%^yef| z;jQ!uk**b0L=@s(sRbO=-iSHM5G^Evec~#7<_B#dpG5a#X~m}@cet{+iDM`?g-QL8 zR5`$TquFR8DqoM#df$49KOkdq`&;*s!!##}yjMzPVyFjo*W8n<^*ndOu`I4gCiaFF zfNkte74MC(gS)Q?GmE;6;=) zm%QxTn_)^_M3duukyrfv-=l2J!yM2iwK5FPt8r3w46%%gxP( zpv#a=Ou(gLvaTK$H^dXMk9-?!G85xqb0Eqj9;A%oFBuP%=zeaEx}Xxu_x3a^0KNON zrEST$p!*s16dM^{o2>CGpd>8SXQ-=r)N7kKrA!vG!p8aVHJUKTPCozV=^`l8FSB{Z zF;>+^h4y=25(x=?TQ{)JH4)i=l@9G&i_3vcWNsg|0L9DZ&1!n}u4Qb{Dd_!X63 zIzK*NXCG=z0ov?b2vakw=X-`P&*&?ZH&`x3a*@)W+1|pCx7;(T)sC@uEjE=s$FDc8K{^36+c@MFSNvw9t$) zhsV05#wjqAT*i?PHT)CQ{>mA$&f1 z$%V&lq#zOj!&qQHI(~vK{}OVgBYhr>w}^;JYA@oP_B53GnRf68d{M@Tj5;z+z8?A* z(e1z?$;Te1Ep1e$7rWHmV^oWT^IC@DFmXcbeedR0jj0a%Vr`2njBkD}m?1WdxC#&n z51;1$)~PeL0@X&pN+ZRe?|+#ediS4gZVfV~0UaGlrGSOll6tl~AC9WW#9n^%o!12} z4^^o5N|(3pEElX}VQA5O597vG-=I`TBKjE=;5jzR6Jvi!)Z;kcEq10j1sct7nK0d_ zofA|XUD z?OT0)$KcJ;bShd7wA7t0DfUmDNK$ntol&t<-iXv(W(vHQOhlTDca?adIJD>a2zC%B zOdM{0E{JP8y_xQ3j8{d=;lDN|HcWv$VEhiCWGv){*a!^QF;uoWP-YU30P~S z!n_E=Y~i};tb1H_(0n*<@+p7$&sMMHk=^bvh$7YpDf(LgfqMSN{|lgz+d8S|NI(C7iYH9s~lq9aexjoZdZE1vH68X)COi|Wn= z@$unn+5}>VhxxjRLf1UDu7^e)Px9@ErlO8DXVsY`QEcWbn}!eyb0JGH2p=C8fnH8o zrm>t4op^GTe3Bl5rTaGEE&S`4{>U1UlCb4%AAvTfAIM+d3Cv84q2wHo%yX7;5i<&g z0Esw&EAvNqpW|gZXmNwN$T$5gXRZV+Nz)WovBp%4zyDlay<%)1`#|InJV?&pdKWIB zW+7-l`nkTsBKc83^!VS30!X+!L_99VcuGdafox)unIv7Gq-A{kv{chDyv{tXljKF- zL<{?Hg}g+b9!0Bpy-ogsMN@MfZQzaZ8?qdn0$lD*Z{ZsIisK+fc*wm5&e*+ni3@`z zQ^d9?pK@}m+^>m+YRenGZ2ZJx$rC9tr40(1uSIOexKH$A_|3U{Yj_LCZD~D!%#43% zePq?0L4*QS>cV|fBCbwnRV_+63PY6NbkRIRW#0IB4~svZ^LDSnF=F`I0OF(J!Ib;A zV0i(s`X!NsMGA-ilt@y^`Wti%hQaB($e#WX#vv_V!_uPrEJr|~04-7#e3g1*ePy2) zTT6vs<~f;|vgp2LJk$T&#ouI$tU_)l#)gi^sNSwR8*x^!9%ai41XMTp!Nelg)lHCygE@so52Z(C>4>24M-lc6&Fx+>#0xXvvIwph1l9r z`L?K@evp>Gb(KMV9MJjwf8x>LPw@zPw7H;yH6hMBNt+~>U@f~)k}xm4Q04S<*t15K z>+PFn7qFh}Oz%gf?2|i51<^0!#A7UssT$xIjgt8;^x2I_4tEhT)I06J;0Yc^Zcu1x zENa_77o5l%zW$_bBkTAM1&?v3684D7N0`fQ_!1hcKA$-?N8MgeXEJL;1iQ{=;pf4U zsqAfD5NLjDpr8L+7-I(g_MwBNsiTvjk*SrbF=&o9zdAN?QU3-2D4#L|4g59mjuFvG z!!caJR2S2zCRCKe{1R2y%)Gn?h}C)`O-)(#U*P$&oRB35Y>BwUg_cUUiP@`baXPiH zQ}w+lHn{1e{emqj`LQ3?jqnq;u<3Zzwq5a$SuOO>*`GBGdIgNLhg&Sa4Yl+dbs4&A zzc03q#-#53HaW#awQ)efaKg?xGh!!rmA>yPfSx>pAy@<0ym68=#ecn)`F(P?ZsXm; znuFRNvUy4r2TmGd#f!xc6XFZ!S+07ziSjjc{OM0_!(pi&%VP{g#`6cW#^1_+I|thr z07u7PDg&sUbGuSN?VNu%o)-t<3?e$+OoGMr7uCC{Do;f^>Ar3^0T~K({lWag(Q2~} zgUdsU3-W8z4-j`X;*vOrIp3c8$-Qc|pz?)4fqd&{_Bl?$tgH*kSw*Ppgx`q@b0wwW z)oDeCIiFHiD>4(_BfZe6@a;(k9tw^TiR5ifX_d4&88${jsUuOA^-edB(&ZN)T*_~T z@%6kCg`Zf?JYvoir+;rQnR>#R{6u=3o}b9?GUYae`zd=ANpe zl)j*+KWA9Fq6UNpJ?Hu^>d{wojJPnlP}I0NUNvC6ylcATY2+P(J}g2Elj9@hh}qcAineopjQ zhN7Y!qb6Mqwa$lXzNJSB79J#C5iDb+t$bwNoXJ#(2pXmYGiOiNM@K0nH0?iLz7(!e zP&FGdLysw_^TXU%JfV3Pzne1`?P%o`7ggQw2(Q~hH1%RUxAqI%rj{h6WwcMVS!-PW zRY!kSOT2|qkNwXrnO89bPzEC4;b!UI8a&2^4raFccD7a^&EELe0UM+$fDYKAKz`^X z=jJy+JJZm1R`wXO39u;+*(V$9O^I)+D_O?sA(p<}WL#FF{jC4hLRZn#lRxZg;r0nc zwONGJf1UN9F`HbmGcF;duyeC#KK4>I6_#j>3%Hk2XYSjw`?BD4BsPdm4EQ=+jC;~-HgBAdTKjW zvZjK5KsX=Pk&Hx#jzcLpSeD8&o<2Bkg3h1+vt8QvERgqBRakp)H~L%A;$UcF3G#3J zIu!8VR=xhNa_BDu5baQ@x(o>F-V?0R362);eFBVED?vlb*b6KiX}Yfq(~jbyO=^aH zGrJ~;;&2g&vZQ)iLNrFJ@OY&@J3b(R>yXrUp`(4xcbtl=nsFh2(6{}AT$48C8+Uje zN{yVaDm44H`|@m2F4SS0x+x*`>#&uWszFf*HCEd%T8E>63@+5M9vLCpK(*rC?=e+D z$PKO531x!(87B0>t?qHV@)5e8S`K7(UpM5EEoyVI=h_T4s_Ml#ybo~}NdYppw+VW_ zT1;*b#E{g337Qe~7Z2@cztjg%DE&G=#znOwfM^1pokjcRn%}6mr-;jwDb2|E)Zkm^ z$E-FsPCGL{Pb%QhkG?^btW!V;#X+{nYY^CZHt-6thXx9ubXDR?G%M(MeBCehX0{NZ z0CKcD53a;4w;DaJoLu@g3Z>TQC^fAKTVa~T+99>@29old>UzGlZ_H}L>QLWg1J&11 z8$wRaDdJ}&E-`1Z1DoW$NUht)2?GJugEbO0UVFAFB=%-c86AEU3~Y!6OmrCHj^e`5 zJD6kfkiOht%piC#V$(E0X2GX_OOv_uZv8DyOb^oZ*IZ+0^sBc3s`mb;+f_^>o)GJ@ zGl}6NNBJ05OyM2r301IA$*MEkUf(=Tsxb&fv}>CSD{J|Z$NC~s3}FL3$Sh5?lyEpa z0Ux=(DGa-dywG)+osWf!TiDo2$q~9C!f6nk&}`l0kpXNlvm$Nz7H%QD{C?@~(9jNg z{%P^?LdVN$35qa*Bx76L$GLK?WtZ&ULZ>7vN2G7BdA6ONw#p2)V$Mq$P_rWh%uCj7 zJYtJck=t9>5&bwTa7%^&ewF(N5E(5GuByLjb#{gT$6rh8mjsaxl4tbV%FOIcbwZs+ z;q$s{Y5d&q64EIPjW8F;T&h>kYjg$Rl57~i5K5bVQ&OD?=}gdDd54FA^$&?mK#WX zaC8LN+WbS(rml3vVh?)Dg*w|A1Jp(AfewUpyfq$LbrAG68mtKF+|vomXj$hNOvwPR z&^2FDZ~<$FzBn%i)u|1>YKphrX?pfCA#9y#t14c)x<_=^mq}>il?)qE-*dh{YC>N# zJI7_h-E;O%h-krjCq-$Zw38hgm$)wxTtUZgve~9rY5CVO(-nG&um+pr}6QT zz(Z*PU5M(??c7<68bbq}i>s8hNfmK=l->{ysv311YL+APE>`N`v~?mfHXb!dzoK@b zE-UqP;X~kkLYGUkAw&6neYpKmT!%djOoQ@Eme9R-vC8T+Xyc1g^ke{GzQJ5*w`b_| z0}z>7w1+*T+T~7Ggd}O$YC&<{)>ujC)VMTSE8X3a%uMX?i;U6ag`UAYqIv8e+OQ0a zz{nvU{au(++n=J0Hq&Kl%5cipYBiftTQN&lGP|kE z5>P86LW;K|jOI$$D_x-|Ne4jKmOo7rHJ=Sbm!-6-bTl0TH+q4g`o>Kb`b!pLhxHHZ zBE2LB^6O?WpJnAWCG7qsOYYzSgJk*w)H%3vwl~;G_-@D~HY-i7bYFw%h2bWV_9gm# zp5&6LN&^87otB3PSeHy95oOaQ4y4MWD5+TV--Wmi-Raj_!Rb~f^V&KwHbm)koWmGi z|De$D1NXEKJv<$X_k+|W6|!wpEc7h#WOoZXXxMYlir5L7HXO~YBmSrScP!vUdgxFkq~9Vh4WE!7oAV7z}LfC({ak zH`pPISPQXnK)k|{`;?McKZe{b_Bf;sNx8%4TMmngQZ#+8sZw(t;`en(JHebi$TZ99 zVD&Fgw&1?Lf!qaG_A{2NQ!NU1M(yCWG^Pp|7I+Nl@M2s{!KZcoxDaL4Cp6Y3dx>DY zStQ4ls0-S{9(*Cc!wF(!q|Y(apb!{rN@0e#E$&q%gR`H(yxQEy!BM zwcvJi^c%%sDsseC1lpOy98=BsxqdU*|4`3m2FXKzA{F%3s5~Z*HWo7pYsxDcA~)|N zTDP5H%p(L3l0N)EAalP zj?EM1{Zm_Z45YG+vha%J@CjR@UW+tcG{eu`V(#eO?3w@hs|w zJT5i}Neu#v=Bb?{c$0;rC-i8Bw94H0{-{4xmZ2$KI&Y{!iApMQ2490Kg36iYh#LIh zL^^jMGo#0qwa*uvpt-B(T?VVZxP=XXwom&w+SFxgc-r0`D95QKj0AcsuQf_a11+nP z_m+vQoL%a=kKVqq3vscT!CIICN^Qf(jf!WghIM3aec8Sik|$j<1s zE+ysmsTz-!1rd^OIJe?3`*DxN02E0`JYKWVelvQ!KYPjEGTzd&oF(Y82I2)>0!p4k z`s#NHZ9~`K0rPbZ zrN?&p^J~U)S=*}%;u;$SE&4ODq>6f11evp3T2t|oCyLiYgHi>U29rf|@2=ivT7wUM z+MourWUvk+7xjA`A*5plmhyah5<{4Mvpja+;n@!%owzx9qi|tU<8GzCT#fMsKprg2 zrvJ3`2T{Epp!X(V3=4hylU|%0pTy_A0{mgy2p4*%Qkt-JVSVv~ zWkCax*gD49&523L-W0{Bb`itn3A24@WtVmxC~27KBz{Vpx_je@v?RmhP6$?1`5&=f z%qJ9%yt8gAKTuR;<}s;shv!9cZyh1Y_h%(F#T`b=4cU8^U-}OJc&Ap?dyz66X|1v! zcQX2uTP<7NYI;M}S3R#g+M7j{_ z*k0=LnZN%~zEq~-6JYin;vx8@w?#T*^an0oq%u^<{)*OIzKc`ka zy(YEo#-Lo}4bV^S62gYXVGS)hByyeSl`+}j<3l|v#H7gpEZHyc+;jAkPMq1vDPIiD zwBh;0KQYaISMcdiHRg?sS8vdLX>ABaSe!n zgAbeMK7bn|CMSVh?x_s^=4md%mr^P%1i7w@a)%Xe>Q@6FXp8*#O)9&=3t=oA4~s2L zG!CAnXi`>M2o0LFF>m3m22C4{*e_G1YOAqO8I2*GKwIKh%Q_#T2|P<LR;1bD( zAOqcO*oPLC4$;w#Q?E+fw&)c<`*`^VrV&5RkSi@Tu{|u(hk9D@F<2d`r0ei28gBrC z=P;>}#OtfCvb`UGF!SW{_r_E{?#ZI+%@jZe!th|h)6 zj6sBFSM-LNd1s0pM=;jgUb0FpGI9`AB~xbPfOH^B z#!0QYvx6$MwBHK7dXqZy*hUv_qH`0M!i}pk8?GP$iw7}@3zfT}D`qK7y%Cu^`YO3^ zj81!ZXx3aY(FY0ThA$D2pOpfwARg@H5DRJhU3OjWO=Y#hs(Gs>Z#!9vyLVwUQmAW1 z`A3Tp)l!z&Gy}7t6hPY5>!6BLkp}L%n)4*}PF@KJb}$FC7ZtFWmn-5XSW|CRUX_{J z>u(2;9_n~^GI?AZURARd<9PcDcANW$#^7D^u9#DNWn#T-54QhxEM$qm%~sxOg#mxK1v-` za?27^TQV2Qqc+4@R3@__W3xj-V?Rsa`YO)E+>))H5!xrvsor||)Ab-Z`mGNQ6{IzJ z-)b@75LjTpuVdZ4z;t^Z>(B8Yy{!LT&bkkD_x{gqOXK(00e$=rpnu(i{|Z?5A@5#X zxosK!9><`L%0rO9uMyqHynCDDR>c23LP7UC9>TnH0p>pF-TM)@lHKny0J3vE1p3Fd zko&NA?;G4&d4G>#=!aqNUA4FmeAi!pYmE9mreGchzJGb)KJs0A`fa=T_Yi=482PSS z|32zn>-V3rr3wEq>c2+!`;d1H)PGW-jPNkzeFyn{(R(_9Cl)vEq|3=#T;CIcE ze?IgeD9t^z-t7Kxq29;6YbE>>mj(S{+&_(%_i^v~?fwilON_ta{%K3Rk9^nAcI(&s zJ<>5BM*g3B?>^#Pci5j%mVosz;vHMuebBq6sy{P%D)z&mzinpsG4HyN{(R5|oQE;* zn6vJK-gWQ%`JkV09|rwxXS$Dh*8%fqPCUnZ81t?R=sxOQ1IeFHia_u%>TfU2eayS2 zjX(D>Qc!L1(7O7=h;kqHuDRjX@$q|r6Fv<4U;dB#=y#m~e=b^PqKDD{v`gH_y(^pF z`U-vzMdF8X|I=k~AO5Z={pb4&ef%)|ZzcXdX5Sqd_s^soM)EM`e~8=n!S5=de}?fp z(ucwC>caO??+S*u>gn$>M)ol3|EHzihrTPn{mJlmkeKvPG&}yQKi Date: Wed, 22 Aug 2018 16:59:50 -0300 Subject: [PATCH 18/23] Started working on the csv file generation --- benchmark/datafiles/train_config.yaml | 303 +++++++++++++++++++++++++- benchmark/nlu_benchmark/cli/run.py | 17 +- 2 files changed, 313 insertions(+), 7 deletions(-) diff --git a/benchmark/datafiles/train_config.yaml b/benchmark/datafiles/train_config.yaml index eeb4c69..05c9269 100644 --- a/benchmark/datafiles/train_config.yaml +++ b/benchmark/datafiles/train_config.yaml @@ -3,5 +3,304 @@ train: - file: /Users/periclesjr/Documents/Benchmark/bothub/benchmark/datafiles/Benchmark_Training.zip - exported_from: witai - filetype: zip -test: - - file: source \ No newline at end of file +tests: + - text: O Real Madrid jogará hoje? + expected: + - intent: esporte + - text: Em que colocação está o Botafogo? + expected: + - intent: esporte + - text: Quais partidas da rodada atual serão televisionadas? + expected: + - intent: esporte + - text: Quantos gols o Barcelona fez no Liverpool? + expected: + - intent: esporte + - text: Quem é o favorito do brasileirão? + expected: + - intent: esporte + - text: Neymar está jogando em que time? + expected: + - intent: esporte + - text: Qual time revelou o jogador Firmino? + expected: + - intent: esporte + - text: Quando vai acontecer o próximo amistoso da seleção brasileira? + expected: + - intent: esporte + - text: Quem foi o maior goleador da história? + expected: + - intent: esporte + - text: Quem foi o melhor jogador do mundo Pelé ou Maradona? + expected: + - intent: esporte + - text: Tem alguma lanchonete aberta? + expected: + - intent: comida + - text: Qual o fastfood mais próximo da minha casa? + expected: + - intent: comida + - text: Qual o cardápio do restaurante Divina's? + expected: + - intent: comida + - text: Onde posso comer hamburguer hoje? + expected: + - intent: comida + - text: Qual a cafeteria menos lotada na sexta feira? + expected: + - intent: comida + - text: qual a faixa de preço do gbarbosa? + expected: + - intent: comida + - text: O Careca abre segunda de 19:00 horas? + expected: + - intent: comida + - text: Quanto custa a pizza de frango? + expected: + - intent: comida + - text: Quais os melhores rodizios da cidade? + expected: + - intent: comida + - text: Qual o delivery de sushi mais próximo da minha residência? + expected: + - intent: comida + - text: Quais filmes de ação você me recomendaria? + expected: + - intent: cinema + - text: Quem está estrelando uma noite no museu? + expected: + - intent: cinema + - text: Quais os melhores documentários sobre astronomia? + expected: + - intent: cinema + - text: Você já assistiu todo mundo em pânico? + expected: + - intent: cinema + - text: Quais os clássicos de terror? + expected: + - intent: cinema + - text: Quais os melhores filmes de steven spielberg? + expected: + - intent: cinema + - text: Ainda existem locadoras de dvd? + expected: + - intent: cinema + - text: Atores mais cotados para fazer Jurasic Park? + expected: + - intent: cinema + - text: Brad pitt atuou em quantos filmes de drama? + expected: + - intent: cinema + - text: Quando foi lançado desventuras em série? + expected: + - intent: cinema + - text: Quantas pessoas nascem na indonésia a cada ano? + expected: + - intent: geografia + - text: Qual o índice pluviométrico da carolina do norte? + expected: + - intent: geografia + - text: Quantos países fazem fronteira com o Cazaquistão? + expected: + - intent: geografia + - text: Qual a cidade mais quente da Nigéria? + expected: + - intent: geografia + - text: Qual pais com maior volume populacional do mundo? + expected: + - intent: geografia + - text: Qual é a área do Congo? + expected: + - intent: geografia + - text: Vai chover hoje? + expected: + - intent: geografia + - text: Quantas pessoas vivem hoje em Alagoas? + expected: + - intent: geografia + - text: Qual a taxa de desemprego no Brasil? + expected: + - intent: geografia + - text: Quantos continentes existem no planeta? + expected: + - intent: geografia + - text: Preciso encontrar um bom psicólogo + expected: + - intent: saude + - text: Qual médico eu deveria consultar para examinar os meus olhos? + expected: + - intent: saude + - text: Estou sentindo uma forte dor nas costas o que devo fazer? + expected: + - intent: saude + - text: Contraí uma gripe pesada devo tomar vitamina C? + expected: + - intent: saude + - text: Meu filho está engasgado como devo proceder? + expected: + - intent: saude + - text: Me feri com uma tesoura posso contrair tétano? + expected: + - intent: saude + - text: Preciso me vacinar contra gripe. + expected: + - intent: saude + - text: Onde fica o posto de saúde mais próximo? + expected: + - intent: saude + - text: Existe alguma clínica 24h na minha cidade? + expected: + - intent: saude + - text: Como posso descobrir se estou diabético? + expected: + - intent: saude + - text: Eu posso pagar em uma lotérica? + expected: + - intent: financeiro + - text: Posso pagar o boleto após o vencimento? + expected: + - intent: financeiro + - text: Posso fazer adiantamento do valor devido? + expected: + - intent: financeiro + - text: tem problema em efetuar o pagamento amanhã? + expected: + - intent: financeiro + - text: É possível atualizar a data de vencimento? + expected: + - intent: financeiro + - text: Eu gostaria de cancelar a minha compra + expected: + - intent: financeiro + - text: Você poderia estornar a diferença da minha compra? + expected: + - intent: financeiro + - text: Existe a possibilidade de financiar o restante da dívida? + expected: + - intent: financeiro + - text: Eu já paguei + expected: + - intent: financeiro + - text: Não quero mais receber essas cobranças + expected: + - intent: financeiro + - text: Tô querendo moto modelo mais novo + expected: + - intent: automoveis + - text: Compro ou troco carro usado + expected: + - intent: automoveis + - text: Vende-se moto honda 150 + expected: + - intent: automoveis + - text: Quando sai o novo lançamento do FIAT + expected: + - intent: automoveis + - text: Quero fazer manutenção no meu automóvel + expected: + - intent: automoveis + - text: Troco carro uno 98 por moto + expected: + - intent: automoveis + - text: Economizei para trocar de carro esse ano + expected: + - intent: automoveis + - text: Compro carro que não seja caro + expected: + - intent: automoveis + - text: Quero comprar o modelo atual do Civic + expected: + - intent: automoveis + - text: Estou interessado na CB300 + expected: + - intent: automoveis + - text: Como faço pra votar? + expected: + - intent: politica + - text: Posso escolher mais de um presidente? + expected: + - intent: politica + - text: Tenho dúvida quanto ao canditado Péricles + expected: + - intent: politica + - text: O candidato 15 é ladrão + expected: + - intent: politica + - text: Vou votar em ninguém esse ano + expected: + - intent: politica + - text: Meu voto será pro 51 + expected: + - intent: politica + - text: Prefiro o Partido Roxo + expected: + - intent: politica + - text: Canditado 51 é ladrão vou votar no 15 + expected: + - intent: politica + - text: Qual o político menos ladrão pra votar? + expected: + - intent: politica + - text: Peter para presidente do Brasil + expected: + - intent: politica + - text: Tô afim de começar um jogo novo + expected: + - intent: games + - text: Tô com task vou jogar só amanhã + expected: + - intent: games + - text: Posso jogar com vocês? + expected: + - intent: games + - text: Esse game pega Multiplayer + expected: + - intent: games + - text: Dá pra logar no PUBG? + expected: + - intent: games + - text: Tô esperando você ficar online pra gente jogar + expected: + - intent: games + - text: Qual o seu game favorito? + expected: + - intent: games + - text: No PC eu uso a Steam + expected: + - intent: games + - text: Já jogou algum jogo da Blizzard? + expected: + - intent: games + - text: Gosto de jogar Minecraft no celular + expected: + - intent: games + - text: Como tratar as pulgas do meu Javalí? + expected: + - intent: pets + - text: Meus cachorros não se dão bem + expected: + - intent: pets + - text: Minha gata está vomitando muitas bolas de pelo + expected: + - intent: pets + - text: Meu cachorro não está muito bem de saúde + expected: + - intent: pets + - text: Estou criando um novo filhote de urso + expected: + - intent: pets + - text: Meu porco de estimação está com pulgas + expected: + - intent: pets + - text: Meus pets latem muito como resolver? + expected: + - intent: pets + - text: Quanto tempo vive um gato egípcio? + expected: + - intent: pets + - text: tenho muitos pets como cuidar de todos? + expected: + - intent: pets + - text: posso criar um cachorro em apartamento? + expected: + - intent: pets \ No newline at end of file diff --git a/benchmark/nlu_benchmark/cli/run.py b/benchmark/nlu_benchmark/cli/run.py index 7467f3b..e264328 100644 --- a/benchmark/nlu_benchmark/cli/run.py +++ b/benchmark/nlu_benchmark/cli/run.py @@ -72,8 +72,8 @@ def run(data_file_path, data_file_type='yaml', bothub_user_token=None, else: with zipfile.ZipFile(data.get('train')[0].get('file'),'r') as training_file: with training_file.open('Benchmark_Training/expressions.json') as json_file: - data = json.loads(json_file.read().decode('utf-8')) - for expression in data['data']: + training_data = json.loads(json_file.read().decode('utf-8')) + for expression in training_data['data']: text = expression['text'] wit_entities = expression['entities'] filtered_entities = [] @@ -101,7 +101,7 @@ def run(data_file_path, data_file_type='yaml', bothub_user_token=None, ## Test test_result = [] - + result_intent_test = '' def analyze_wrapper(text, expected={}): analyze_response = bothub.analyze( temporary_repository.get('owner__nickname'), @@ -113,14 +113,20 @@ def analyze_wrapper(text, expected={}): analyze = analyze_response.json() analyze_answer = analyze.get('answer') analyze_intent = analyze_answer.get('intent') + result_intent_test = [] + if expected[0].get('intent') == analyze_intent.get('name'): + result_intent_test = 'OK' + else: + result_intent_test = 'FAILURE' logger.info('Bothub return:') logger.info(f' - intent: {analyze_intent.get("name", "[not detected]")} ' + f'({analyze_intent.get("confidence", 0) * 100}%)') test_result.append({ 'text': text, 'intent': analyze_intent, + 'entities': analyze_answer.get('entities'), 'expected': expected, - 'response': analyze, + 'result': result_intent_test }) if type_tests: @@ -130,7 +136,7 @@ def analyze_wrapper(text, expected={}): text = input('Type a text to test: ') analyze_wrapper(text) except KeyboardInterrupt as e: - logger.info('Tests finished!') + logger.info('Test finished!') else: for test in data.get('tests'): analyze_wrapper(test.get('text'), test.get('expected')) @@ -141,6 +147,7 @@ def analyze_wrapper(text, expected={}): except Exception as e: raise e finally: + print(test_result) ## Delete a temporary repository delete_repository_response = bothub.delete_repository( From 276baa5ca821c8c226fb73adb421c58a00645541 Mon Sep 17 00:00:00 2001 From: periclesJ Date: Fri, 24 Aug 2018 14:26:21 -0300 Subject: [PATCH 19/23] Working on csv file output --- benchmark/datafiles/testPhrases.yaml | 302 ++++++++++++++++++++++++ benchmark/datafiles/train_config.yaml | 6 +- benchmark/nlu_benchmark/cli/run.py | 37 ++- benchmark/nlu_benchmark/services/wit.py | 19 ++ 4 files changed, 358 insertions(+), 6 deletions(-) create mode 100644 benchmark/datafiles/testPhrases.yaml create mode 100644 benchmark/nlu_benchmark/services/wit.py diff --git a/benchmark/datafiles/testPhrases.yaml b/benchmark/datafiles/testPhrases.yaml new file mode 100644 index 0000000..93c2483 --- /dev/null +++ b/benchmark/datafiles/testPhrases.yaml @@ -0,0 +1,302 @@ +language: en +tests: + - text: "O Real Madrid jogará hoje?" + expected: + - intent: "esporte" + - text: "Em que colocação está o Botafogo?" + expected: + - intent: "esporte" + - text: "Quais partidas da rodada atual serão televisionadas?" + expected: + - intent: "esporte" + - text: "Quantos gols o Barcelona fez no Liverpool?" + expected: + - intent: "esporte" + - text: "Quem é o favorito do brasileirão?" + expected: + - intent: "esporte" + - text: "Neymar está jogando em que time?" + expected: + - intent: "esporte" + - text: "Qual time revelou o jogador Firmino?" + expected: + - intent: "esporte" + - text: "Quando vai acontecer o próximo amistoso da seleção brasileira?" + expected: + - intent: "esporte" + - text: "Quem foi o maior goleador da história?" + expected: + - intent: "esporte" + - text: "Quem foi o melhor jogador do mundo Pelé ou Maradona?" + expected: + - intent: "esporte" + - text: "Tem alguma lanchonete aberta?" + expected: + - intent: "comida" + - text: "Qual o fastfood mais próximo da minha casa?" + expected: + - intent: "comida" + - text: "Qual o cardápio do restaurante Divina's?" + expected: + - intent: "comida" + - text: "Onde posso comer hamburguer hoje?" + expected: + - intent: "comida" + - text: "Qual a cafeteria menos lotada na sexta feira?" + expected: + - intent: "comida" + - text: "qual a faixa de preço do gbarbosa?" + expected: + - intent: "comida" + - text: "O Careca abre segunda de 19:00 horas?" + expected: + - intent: "comida" + - text: "Quanto custa a pizza de frango?" + expected: + - intent: "comida" + - text: "Quais os melhores rodizios da cidade?" + expected: + - intent: "comida" + - text: "Qual o delivery de sushi mais próximo da minha residência?" + expected: + - intent: "comida" + - text: "Quais filmes de ação você me recomendaria?" + expected: + - intent: "cinema" + - text: "Quem está estrelando uma noite no museu?" + expected: + - intent: "cinema" + - text: "Quais os melhores documentários sobre astronomia?" + expected: + - intent: "cinema" + - text: "Você já assistiu todo mundo em pânico?" + expected: + - intent: "cinema" + - text: "Quais os clássicos de terror?" + expected: + - intent: "cinema" + - text: "Quais os melhores filmes de steven spielberg?" + expected: + - intent: "cinema" + - text: "Ainda existem locadoras de dvd?" + expected: + - intent: "cinema" + - text: "Atores mais cotados para fazer Jurasic Park?" + expected: + - intent: "cinema" + - text: "Brad pitt atuou em quantos filmes de drama?" + expected: + - intent: "cinema" + - text: "Quando foi lançado desventuras em série?" + expected: + - intent: "cinema" + - text: "Quantas pessoas nascem na indonésia a cada ano?" + expected: + - intent: "geografia" + - text: "Qual o índice pluviométrico da carolina do norte?" + expected: + - intent: "geografia" + - text: "Quantos países fazem fronteira com o Cazaquistão?" + expected: + - intent: "geografia" + - text: "Qual a cidade mais quente da Nigéria?" + expected: + - intent: "geografia" + - text: "Qual pais com maior volume populacional do mundo?" + expected: + - intent: "geografia" + - text: "Qual é a área do Congo?" + expected: + - intent: "geografia" + - text: "Vai chover hoje?" + expected: + - intent: "geografia" + - text: "Quantas pessoas vivem hoje em Alagoas?" + expected: + - intent: "geografia" + - text: "Qual a taxa de desemprego no Brasil?" + expected: + - intent: "geografia" + - text: "Quantos continentes existem no planeta?" + expected: + - intent: "geografia" + - text: "Preciso encontrar um bom psicólogo" + expected: + - intent: "saude" + - text: "Qual médico eu deveria consultar para examinar os meus olhos?" + expected: + - intent: "saude" + - text: "Estou sentindo uma forte dor nas costas o que devo fazer?" + expected: + - intent: "saude" + - text: "Contraí uma gripe pesada devo tomar vitamina C?" + expected: + - intent: "saude" + - text: "Meu filho está engasgado como devo proceder?" + expected: + - intent: "saude" + - text: "Me feri com uma tesoura posso contrair tétano?" + expected: + - intent: "saude" + - text: "Preciso me vacinar contra gripe." + expected: + - intent: "saude" + - text: "Onde fica o posto de saúde mais próximo?" + expected: + - intent: "saude" + - text: "Existe alguma clínica 24h na minha cidade?" + expected: + - intent: "saude" + - text: "Como posso descobrir se estou diabético?" + expected: + - intent: "saude" + - text: "Eu posso pagar em uma lotérica?" + expected: + - intent: "financeiro" + - text: "Posso pagar o boleto após o vencimento?" + expected: + - intent: "financeiro" + - text: "Posso fazer adiantamento do valor devido?" + expected: + - intent: "financeiro" + - text: "tem problema em efetuar o pagamento amanhã?" + expected: + - intent: "financeiro" + - text: "É possível atualizar a data de vencimento?" + expected: + - intent: "financeiro" + - text: "Eu gostaria de cancelar a minha compra" + expected: + - intent: "financeiro" + - text: "Você poderia estornar a diferença da minha compra?" + expected: + - intent: "financeiro" + - text: "Existe a possibilidade de financiar o restante da dívida?" + expected: + - intent: "financeiro" + - text: "Eu já paguei" + expected: + - intent: "financeiro" + - text: "Não quero mais receber essas cobranças" + expected: + - intent: "financeiro" + - text: "Tô querendo moto modelo mais novo" + expected: + - intent: "automoveis" + - text: "Compro ou troco carro usado" + expected: + - intent: "automoveis" + - text: "Vende-se moto honda 150" + expected: + - intent: "automoveis" + - text: "Quando sai o novo lançamento do FIAT" + expected: + - intent: "automoveis" + - text: "Quero fazer manutenção no meu automóvel" + expected: + - intent: "automoveis" + - text: "Troco carro uno 98 por moto" + expected: + - intent: "automoveis" + - text: "Economizei para trocar de carro esse ano" + expected: + - intent: "automoveis" + - text: "Compro carro que não seja caro" + expected: + - intent: "automoveis" + - text: "Quero comprar o modelo atual do Civic" + expected: + - intent: "automoveis" + - text: "Estou interessado na CB300" + expected: + - intent: "automoveis" + - text: "Como faço pra votar?" + expected: + - intent: "politica" + - text: "Posso escolher mais de um presidente?" + expected: + - intent: "politica" + - text: "Tenho dúvida quanto ao canditado Péricles" + expected: + - intent: "politica" + - text: "O candidato 15 é ladrão" + expected: + - intent: "politica" + - text: "Vou votar em ninguém esse ano" + expected: + - intent: "politica" + - text: "Meu voto será pro 51" + expected: + - intent: "politica" + - text: "Prefiro o Partido Roxo" + expected: + - intent: "politica" + - text: "Canditado 51 é ladrão vou votar no 15" + expected: + - intent: "politica" + - text: "Qual o político menos ladrão pra votar?" + expected: + - intent: "politica" + - text: "Peter para presidente do Brasil" + expected: + - intent: "politica" + - text: "Tô afim de começar um jogo novo" + expected: + - intent: "games" + - text: "Tô com task vou jogar só amanhã" + expected: + - intent: "games" + - text: "Posso jogar com vocês?" + expected: + - intent: "games" + - text: "Esse game pega Multiplayer" + expected: + - intent: "games" + - text: "Dá pra logar no PUBG?" + expected: + - intent: "games" + - text: "Tô esperando você ficar online pra gente jogar" + expected: + - intent: "games" + - text: "Qual o seu game favorito?" + expected: + - intent: "games" + - text: "No PC eu uso a Steam" + expected: + - intent: "games" + - text: "Já jogou algum jogo da Blizzard?" + expected: + - intent: "games" + - text: "Gosto de jogar Minecraft no celular" + expected: + - intent: "games" + - text: "Como tratar as pulgas do meu Javalí?" + expected: + - intent: "pets" + - text: "Meus cachorros não se dão bem" + expected: + - intent: "pets" + - text: "Minha gata está vomitando muitas bolas de pelo" + expected: + - intent: "pets" + - text: "Meu cachorro não está muito bem de saúde" + expected: + - intent: "pets" + - text: "Estou criando um novo filhote de urso" + expected: + - intent: "pets" + - text: "Meu porco de estimação está com pulgas" + expected: + - intent: "pets" + - text: "Meus pets latem muito como resolver?" + expected: + - intent: "pets" + - text: "Quanto tempo vive um gato egípcio?" + expected: + - intent: "pets" + - text: "tenho muitos pets como cuidar de todos?" + expected: + - intent: "pets" + - text: "posso criar um cachorro em apartamento?" + expected: + - intent: "pets" \ No newline at end of file diff --git a/benchmark/datafiles/train_config.yaml b/benchmark/datafiles/train_config.yaml index 05c9269..c4af394 100644 --- a/benchmark/datafiles/train_config.yaml +++ b/benchmark/datafiles/train_config.yaml @@ -1,7 +1,9 @@ language: pt_br train: - - file: /Users/periclesjr/Documents/Benchmark/bothub/benchmark/datafiles/Benchmark_Training.zip - - exported_from: witai + - filepath: /Users/periclesjr/Documents/Benchmark/bothub/benchmark/datafiles/Benchmark_Training.zip + - filename: 'Benchmark_Training' + - wit_token: 'Bearer OWL5XSVZVJNJS4BAVLVXV6STDU7SMPUF' + - wit_version: 20180808 - filetype: zip tests: - text: O Real Madrid jogará hoje? diff --git a/benchmark/nlu_benchmark/cli/run.py b/benchmark/nlu_benchmark/cli/run.py index e264328..04cefef 100644 --- a/benchmark/nlu_benchmark/cli/run.py +++ b/benchmark/nlu_benchmark/cli/run.py @@ -8,6 +8,7 @@ from . import logger from ..services.bothub import Bothub +from ..services.wit import Wit @plac.annotations( data_file_path=plac.Annotation(), @@ -40,6 +41,7 @@ def run(data_file_path, data_file_type='yaml', bothub_user_token=None, logger.info('Starting Bothub benchmark...') bothub = Bothub(user_token=bothub_user_token) + wit = Wit() ## Create a temporary repository @@ -59,7 +61,7 @@ def run(data_file_path, data_file_type='yaml', bothub_user_token=None, try: ## Train train = data.get('train') - if train[2].get('filetype') != 'zip': + if 'filetype' in train[2] and train[4].get('filetype') != 'zip': total_train = len(train) logger.info(f'{total_train} examples to train in Bothub') with tqdm(total=total_train, unit='examples') as pbar: @@ -70,7 +72,7 @@ def run(data_file_path, data_file_type='yaml', bothub_user_token=None, pbar.update(1) logger.info('Examples submitted!') else: - with zipfile.ZipFile(data.get('train')[0].get('file'),'r') as training_file: + with zipfile.ZipFile(data.get('train')[0].get('filepath'),'r') as training_file: with training_file.open('Benchmark_Training/expressions.json') as json_file: training_data = json.loads(json_file.read().decode('utf-8')) for expression in training_data['data']: @@ -103,6 +105,7 @@ def run(data_file_path, data_file_type='yaml', bothub_user_token=None, test_result = [] result_intent_test = '' def analyze_wrapper(text, expected={}): + analyze_response = bothub.analyze( temporary_repository.get('owner__nickname'), temporary_repository.get('slug'), @@ -120,7 +123,7 @@ def analyze_wrapper(text, expected={}): result_intent_test = 'FAILURE' logger.info('Bothub return:') logger.info(f' - intent: {analyze_intent.get("name", "[not detected]")} ' + - f'({analyze_intent.get("confidence", 0) * 100}%)') + f'({int(analyze_intent.get("confidence", 0) * 100)}%)') test_result.append({ 'text': text, 'intent': analyze_intent, @@ -129,6 +132,9 @@ def analyze_wrapper(text, expected={}): 'result': result_intent_test }) + analyze_wit_response = wit.analyze(text,data.get('train')[3],data.get('train')[2].get('wit_token')) + print(analyze_wit_response.text) + if type_tests: logger.warning('Typing mode ON, press CTRL + C to exit') try: @@ -147,7 +153,30 @@ def analyze_wrapper(text, expected={}): except Exception as e: raise e finally: - print(test_result) + + ## Write CSV file with test results + + logger.info(f'Writing CSV file...') + csv_headers = "Phrases,Expected intents,Bothub predicts,Confidence accuracy,Result by Intent,Detected entities,Known entities,Result by Entity, Entity accuracy\n" + with open('Bothub_output.csv', "w") as csv_file: + bothub_hits: int = 0 + bothub_failures: int = 0 + csv_file.write(csv_headers) + for example in test_result: + entities = '' + if len(example.get('entities')) > 0: + entities = '|' + for entity in example.get('entities'): + entities += '{0}=>{1}|'.format(entity.get('value'), entity.get('entity')) + if example.get('result') == 'OK': + bothub_hits += 1 + else: + bothub_failures += 1 + csv_file.write('{0},{1},{2},{3}%,{4},{5}'.format(example.get('text'),example.get('expected')[0].get('intent'),example.get('intent').get('name'),int(example.get('intent').get('confidence')*100),example.get('result'),entities)) + csv_file.write('\n') + csv_file.write('\n' + 'Analized phrases: {0}\nSuccess average: {1}%\nWrong predictions: {2}'.format(len(test_result), bothub_hits,bothub_failures)) + logger.info(f'Bothub_output.csv saved!') + ## Delete a temporary repository delete_repository_response = bothub.delete_repository( diff --git a/benchmark/nlu_benchmark/services/wit.py b/benchmark/nlu_benchmark/services/wit.py new file mode 100644 index 0000000..9a8f732 --- /dev/null +++ b/benchmark/nlu_benchmark/services/wit.py @@ -0,0 +1,19 @@ +import string +import json + +from requests import Request, Session + +from .service import Service + + +class Wit(Service): + GLOBAL_CONFIG_FILE = '.bothub-service.yaml' + + @classmethod + def analyze(self,phrase,version,token): + session = Session() + params = {'q':phrase, 'v':version} + headers = {'Authorization':token} + request = Request('GET','https://api.wit.ai/message',params=params,headers=headers) + prepped = request.prepare() + return session.send(prepped) \ No newline at end of file From 57d43a6b02aa5cd28fe7dbb61edc9b93f6bed8f8 Mon Sep 17 00:00:00 2001 From: periclesJ Date: Thu, 30 Aug 2018 08:26:35 -0300 Subject: [PATCH 20/23] bothub report finished --- benchmark/datafiles/train_config.yaml | 273 +++++++++++++++++++++++++- benchmark/nlu_benchmark/cli/run.py | 87 ++++++-- 2 files changed, 346 insertions(+), 14 deletions(-) diff --git a/benchmark/datafiles/train_config.yaml b/benchmark/datafiles/train_config.yaml index c4af394..1d22937 100644 --- a/benchmark/datafiles/train_config.yaml +++ b/benchmark/datafiles/train_config.yaml @@ -9,168 +9,351 @@ tests: - text: O Real Madrid jogará hoje? expected: - intent: esporte + - entities: + - value: Real Madrid + entity: time - text: Em que colocação está o Botafogo? expected: - intent: esporte + - entities: + - value: Botafogo + entity: time - text: Quais partidas da rodada atual serão televisionadas? expected: - intent: esporte - text: Quantos gols o Barcelona fez no Liverpool? expected: - intent: esporte + - entities: + - value: Barcelona + entity: time + - value: Liverpool + entity: time - text: Quem é o favorito do brasileirão? expected: - intent: esporte + - entities: + - value: brasileirão + entity: competicao - text: Neymar está jogando em que time? expected: - intent: esporte + - entities: + - value: Neymar + entity: nome - text: Qual time revelou o jogador Firmino? expected: - intent: esporte + - entities: + - value: Firmino + entity: nome - text: Quando vai acontecer o próximo amistoso da seleção brasileira? expected: - intent: esporte + - entities: + - value: seleção brasileira + entity: time - text: Quem foi o maior goleador da história? expected: - intent: esporte - text: Quem foi o melhor jogador do mundo Pelé ou Maradona? expected: - intent: esporte + - entities: + - value: Pelé + entity: nome + - value: Maradona + entity: nome - text: Tem alguma lanchonete aberta? expected: - intent: comida + - entities: + - value: lanchonete + entity: lugar - text: Qual o fastfood mais próximo da minha casa? expected: - intent: comida + - entities: + - value: fastfood + entity: lugar + - value: minha casa + entity: lugar - text: Qual o cardápio do restaurante Divina's? expected: - intent: comida + - entities: + - value: restaurante + entity: lugar + - value: Divina's + entity: lugar + - value: Divina's + entity: marca - text: Onde posso comer hamburguer hoje? expected: - intent: comida + - entities: + - value: hamburguer + entity: tipo_comida - text: Qual a cafeteria menos lotada na sexta feira? expected: - intent: comida + - entities: + - value: sexta feira + entity: data + - value: cafeteria + entity: lugar - text: qual a faixa de preço do gbarbosa? expected: - intent: comida + - entities: + - value: gbarbosa + entity: lugar + - value: gbarbosa + entity: marca - text: O Careca abre segunda de 19:00 horas? expected: - intent: comida + - entities: + - value: Careca + entity: lugar + - value: segunda + entity: data - text: Quanto custa a pizza de frango? expected: - intent: comida + - entities: + - value: pizza de frango + entity: tipo_comida - text: Quais os melhores rodizios da cidade? expected: - intent: comida + - entities: + - value: cidade + entity: lugar - text: Qual o delivery de sushi mais próximo da minha residência? expected: - intent: comida + - entities: + - value: delivery + entity: tipo_comida + - value: sushi + entity: tipo_comida + - value: minha residência + entity: lugar - text: Quais filmes de ação você me recomendaria? expected: - intent: cinema + - entities: + - value: ação + entity: genero + - value: filmes + entity: tipo_de_programa - text: Quem está estrelando uma noite no museu? expected: - intent: cinema + - entities: + - value: uma noite no museu + entity: tipo_de_programa - text: Quais os melhores documentários sobre astronomia? expected: - intent: cinema + - entities: + - value: documentários + entity: tipo_de_programa - text: Você já assistiu todo mundo em pânico? expected: - intent: cinema + - entities: + - value: todo mundo em pânico + entity: tipo_de_programa - text: Quais os clássicos de terror? expected: - intent: cinema + - entities: + - value: terror + entity: genero - text: Quais os melhores filmes de steven spielberg? expected: - intent: cinema + - entities: + - value: steven spielberg + entity: nome - text: Ainda existem locadoras de dvd? expected: - intent: cinema + - entities: + - value: locadoras + entity: lugar - text: Atores mais cotados para fazer Jurasic Park? expected: - intent: cinema + - entities: + - value: Atores + entity: profissao + - value: Jurasic Park + entity: tipo_de_programa - text: Brad pitt atuou em quantos filmes de drama? expected: - intent: cinema + - entities: + - value: Brad pitt + entity: nome + - value: drama + entity: genero - text: Quando foi lançado desventuras em série? expected: - intent: cinema + - entities: + - value: desventuras em série + entity: tipo_de_programa - text: Quantas pessoas nascem na indonésia a cada ano? expected: - intent: geografia + - entities: + - value: indonésia + entity: lugar + - value: ano + entity: data - text: Qual o índice pluviométrico da carolina do norte? expected: - intent: geografia + - entities: + - value: carolina do norte + entity: lugar - text: Quantos países fazem fronteira com o Cazaquistão? expected: - intent: geografia + - entities: + - value: Cazaquistão + entity: lugar - text: Qual a cidade mais quente da Nigéria? expected: - intent: geografia + - entities: + - value: Nigéria + entity: lugar + - value: cidade + entity: lugar - text: Qual pais com maior volume populacional do mundo? expected: - intent: geografia - text: Qual é a área do Congo? expected: - intent: geografia + - entities: + - value: Congo + entity: Lugar - text: Vai chover hoje? expected: - intent: geografia - text: Quantas pessoas vivem hoje em Alagoas? expected: - intent: geografia + - entities: + - value: Alagoas + entity: Lugar - text: Qual a taxa de desemprego no Brasil? expected: - intent: geografia + - entities: + - value: Brasil + entity: Lugar - text: Quantos continentes existem no planeta? expected: - intent: geografia + - entities: + - value: continentes + entity: lugar - text: Preciso encontrar um bom psicólogo expected: - intent: saude + - entities: + - value: psicólogo + entity: profissao - text: Qual médico eu deveria consultar para examinar os meus olhos? expected: - intent: saude + - entities: + - value: olhos + entity: parte_do_corpo + - value: médico + entity: profissao - text: Estou sentindo uma forte dor nas costas o que devo fazer? expected: - intent: saude + - entities: + - value: dor nas costas + entity: patologia + - value: costas + entity: parte_do_corpo - text: Contraí uma gripe pesada devo tomar vitamina C? expected: - intent: saude + - entities: + - value: gripe + entity: patologia - text: Meu filho está engasgado como devo proceder? expected: - intent: saude + - entities: + - value: engasgado + entity: patologia - text: Me feri com uma tesoura posso contrair tétano? expected: - intent: saude + - entities: + - value: tétano + entity: patologia - text: Preciso me vacinar contra gripe. expected: - intent: saude - text: Onde fica o posto de saúde mais próximo? expected: - intent: saude + - entities: + - value: posto de saúde + entity: lugar - text: Existe alguma clínica 24h na minha cidade? expected: - intent: saude + - entities: + - value: cidade + entity: lugar + - value: clínica + entity: lugar - text: Como posso descobrir se estou diabético? expected: - intent: saude + - entities: + - value: diabético + entity: patologia - text: Eu posso pagar em uma lotérica? expected: - intent: financeiro + - entities: + - value: lotérica + entity: lugar - text: Posso pagar o boleto após o vencimento? expected: - intent: financeiro + - entities: + - value: após o vencimento + entity: data - text: Posso fazer adiantamento do valor devido? expected: - intent: financeiro - text: tem problema em efetuar o pagamento amanhã? expected: - intent: financeiro + - entities: + - value: amanhã + entity: data - text: É possível atualizar a data de vencimento? expected: - intent: financeiro + - entities: + - value: data de vencimento + entity: data - text: Eu gostaria de cancelar a minha compra expected: - intent: financeiro @@ -189,54 +372,92 @@ tests: - text: Tô querendo moto modelo mais novo expected: - intent: automoveis + - entities: + - value: moto + entity: veiculo - text: Compro ou troco carro usado expected: - intent: automoveis + - entities: + - value: carro + entity: veiculo - text: Vende-se moto honda 150 expected: - intent: automoveis + - entities: + - value: honda 150 + entity: modelo_moto + - value: honda + entity: marca - text: Quando sai o novo lançamento do FIAT expected: - intent: automoveis + - entities: + - value: FIAT + entity: marca - text: Quero fazer manutenção no meu automóvel expected: - intent: automoveis - text: Troco carro uno 98 por moto expected: - intent: automoveis + - entities: + - value: uno + entity: modelo_carro - text: Economizei para trocar de carro esse ano expected: - intent: automoveis + - entities: + - value: esse ano + entity: data - text: Compro carro que não seja caro expected: - intent: automoveis - text: Quero comprar o modelo atual do Civic expected: - intent: automoveis + - entities: + - value: Civic + entity: marca - text: Estou interessado na CB300 expected: - intent: automoveis + - entities: + - value: CB300 + entity: modelo_moto - text: Como faço pra votar? expected: - intent: politica - text: Posso escolher mais de um presidente? expected: - intent: politica + - entities: + - value: presidente + entity: cargo_politico - text: Tenho dúvida quanto ao canditado Péricles expected: - intent: politica + - entities: + - value: Péricles + entity: nome - text: O candidato 15 é ladrão expected: - intent: politica - text: Vou votar em ninguém esse ano expected: - intent: politica + - entities: + - value: ano + entity: data - text: Meu voto será pro 51 expected: - intent: politica - text: Prefiro o Partido Roxo expected: - intent: politica + - entities: + - value: Roxo + entity: cor - text: Canditado 51 é ladrão vou votar no 15 expected: - intent: politica @@ -246,21 +467,35 @@ tests: - text: Peter para presidente do Brasil expected: - intent: politica + - entities: + - value: Peter + entity: nome + - value: Brasil + entity: lugar - text: Tô afim de começar um jogo novo expected: - intent: games - text: Tô com task vou jogar só amanhã expected: - intent: games + - entities: + - value: amanhã + entity: data - text: Posso jogar com vocês? expected: - intent: games - text: Esse game pega Multiplayer expected: - intent: games + - entities: + - value: Multiplayer + entity: tipo_jogo - text: Dá pra logar no PUBG? expected: - intent: games + - entities: + - value: PUBG + entity: titulo_jogo - text: Tô esperando você ficar online pra gente jogar expected: - intent: games @@ -270,30 +505,63 @@ tests: - text: No PC eu uso a Steam expected: - intent: games + - entities: + - value: Steam + entity: marca - text: Já jogou algum jogo da Blizzard? expected: - intent: games + - entities: + - value: Blizzard + entity: marca - text: Gosto de jogar Minecraft no celular expected: - intent: games + - entities: + - value: Minecraft + entity: titulo_jogo - text: Como tratar as pulgas do meu Javalí? expected: - intent: pets + - entities: + - value: Javalí + entity: animal + - value: pulgas + entity: patologia + - value: pulgas + entity: animal - text: Meus cachorros não se dão bem expected: - intent: pets + - entities: + - value: cachorros + entity: animal - text: Minha gata está vomitando muitas bolas de pelo expected: - intent: pets + - entities: + - value: vomitando + entity: patologia + - value: gata + entity: animal - text: Meu cachorro não está muito bem de saúde expected: - intent: pets + - entities: + - value: cachorro + entity: animal - text: Estou criando um novo filhote de urso expected: - intent: pets + - entities: + - value: urso + entity: animal - text: Meu porco de estimação está com pulgas expected: - intent: pets + - entities: + - value: porco + entity: animal - text: Meus pets latem muito como resolver? expected: - intent: pets @@ -305,4 +573,7 @@ tests: - intent: pets - text: posso criar um cachorro em apartamento? expected: - - intent: pets \ No newline at end of file + - intent: pets + - entities: + - value: apartamento + entity: lugar \ No newline at end of file diff --git a/benchmark/nlu_benchmark/cli/run.py b/benchmark/nlu_benchmark/cli/run.py index 04cefef..00da1a2 100644 --- a/benchmark/nlu_benchmark/cli/run.py +++ b/benchmark/nlu_benchmark/cli/run.py @@ -102,7 +102,8 @@ def run(data_file_path, data_file_type='yaml', bothub_user_token=None, ## Test - test_result = [] + test_bothub_result = [] + test_wit_result = [] result_intent_test = '' def analyze_wrapper(text, expected={}): @@ -124,7 +125,7 @@ def analyze_wrapper(text, expected={}): logger.info('Bothub return:') logger.info(f' - intent: {analyze_intent.get("name", "[not detected]")} ' + f'({int(analyze_intent.get("confidence", 0) * 100)}%)') - test_result.append({ + test_bothub_result.append({ 'text': text, 'intent': analyze_intent, 'entities': analyze_answer.get('entities'), @@ -133,7 +134,27 @@ def analyze_wrapper(text, expected={}): }) analyze_wit_response = wit.analyze(text,data.get('train')[3],data.get('train')[2].get('wit_token')) - print(analyze_wit_response.text) + entities = analyze_wit_response.json().get('entities') + + intent_result = '-' + entities= '|' + # for key in entities: + # print(key) + # entities += str(key)+'|' + # print(entities) + if 'intent' in entities: + print('--------------------------Open') + for entity in entities.get('intent'): + # print(entities) + # print('-'+str(entity) +''+entity.get('value')) + test_wit_result.append({ + 'text': text, + 'intent': entity.get('value'), + 'confidence': entity.get('confidence'), + 'expected':expected + }) + print('--------------------------Close') + if type_tests: logger.warning('Typing mode ON, press CTRL + C to exit') @@ -147,7 +168,7 @@ def analyze_wrapper(text, expected={}): for test in data.get('tests'): analyze_wrapper(test.get('text'), test.get('expected')) - logger.debug(f'test_result: {test_result}') + logger.debug(f'test_bothub_result: {test_bothub_result}') except requests.exceptions.HTTPError as e: raise e except Exception as e: @@ -156,27 +177,67 @@ def analyze_wrapper(text, expected={}): ## Write CSV file with test results - logger.info(f'Writing CSV file...') - csv_headers = "Phrases,Expected intents,Bothub predicts,Confidence accuracy,Result by Intent,Detected entities,Known entities,Result by Entity, Entity accuracy\n" - with open('Bothub_output.csv', "w") as csv_file: + logger.info(f'Writing CSV files...') + csv_headers = "Phrases,Expected intents,Bothub predicts,Confidence accuracy,Result by Intent,Detected entities,Expected entities,Result by Entity, Entity accuracy\n" + with open('Bothub_output.csv', 'w') as csv_file: bothub_hits: int = 0 bothub_failures: int = 0 + bothub_entities_count: int = 0 + known_entities_count: int = 0 + matched_entities: int = 0 + entities_percentage: int = 0 + entity_result = '' csv_file.write(csv_headers) - for example in test_result: + for example in test_bothub_result: entities = '' + expected_entities = '' + if len(example.get('expected')) > 1: + expected_entities = '|' + for entity in example.get('expected')[1].get('entities'): + known_entities_count += 1 + expected_entities += '{0}=>{1}|'.format(entity.get('value'), entity.get('entity')) if len(example.get('entities')) > 0: entities = '|' for entity in example.get('entities'): + bothub_entities_count += 1 entities += '{0}=>{1}|'.format(entity.get('value'), entity.get('entity')) + if len(example.get('expected')) > 1: + for known_entity in example.get('expected')[1].get('entities'): + if entity.get('entity') == known_entity.get('entity'): + matched_entities += 1 if example.get('result') == 'OK': bothub_hits += 1 else: - bothub_failures += 1 - csv_file.write('{0},{1},{2},{3}%,{4},{5}'.format(example.get('text'),example.get('expected')[0].get('intent'),example.get('intent').get('name'),int(example.get('intent').get('confidence')*100),example.get('result'),entities)) - csv_file.write('\n') - csv_file.write('\n' + 'Analized phrases: {0}\nSuccess average: {1}%\nWrong predictions: {2}'.format(len(test_result), bothub_hits,bothub_failures)) + bothub_failures += 1 + if bothub_entities_count == known_entities_count == matched_entities: + entity_result = 'OK' + entities_percentage = 100 + elif bothub_entities_count != known_entities_count and matched_entities > 0: + entities_percentage = (bothub_entities_count/known_entities_count)*100 + entity_result = 'PARCIAL' + else: + entities_percentage = 0 + entity_result = 'FAILURE' + + csv_file.write('{0},{1},{2},{3}%,{4},{5},{6},{7},{8}%'.format(example.get('text'),example.get('expected')[0].get('intent'),example.get('intent').get('name'),int(example.get('intent').get('confidence')*100),example.get('result'),entities,expected_entities,entity_result,entities_percentage)) + csv_file.write('\n') + bothub_entities_count: int = 0 + known_entities_count: int = 0 + matched_entities: int = 0 + entities_percentage: int = 0 + entity_result = '' + csv_file.write('\n' + 'Analized phrases: {0}\nSuccess average: {1}%\nWrong predictions: {2}'.format(len(test_bothub_result), bothub_hits,bothub_failures)) logger.info(f'Bothub_output.csv saved!') + with open('Wit_output.csv','w') as csv_file: + wit_hits: int = 0 + wit_failures: int = 0 + csv_file.write(csv_headers) + for example in test_wit_result: + csv_file.write('{0},{1},{2},{3}%,{4},{5}'.format(example.get('text'),example.get('expected')[0].get('intent'),example.get('intent'),int(example.get('confidence')*100),example.get('result'),entities)) + csv_file.write('\n') + csv_file.write('\n' + 'Analized phrases: {0}\nSuccess average: {1}%\nWrong predictions: {2}'.format(len(test_wit_result), wit_hits,wit_failures)) + logger.info(f'Wit_output.csv saved!') ## Delete a temporary repository delete_repository_response = bothub.delete_repository( @@ -189,4 +250,4 @@ def analyze_wrapper(text, expected={}): logger.warning('Bothub temporary repository not deleted, manually delete') logger.debug(delete_repository_response.text) - logger.info(f'Bothub temporary repository deleted') + logger.info(f'Bothub temporary repository deleted') \ No newline at end of file From 57c9e79e1b850933e857dd7c40a7b414c34afef5 Mon Sep 17 00:00:00 2001 From: periclesJ Date: Thu, 30 Aug 2018 16:04:48 -0300 Subject: [PATCH 21/23] Finished script...it needs some test --- benchmark/nlu_benchmark/cli/run.py | 89 +++++++++++++++++++++--------- 1 file changed, 63 insertions(+), 26 deletions(-) diff --git a/benchmark/nlu_benchmark/cli/run.py b/benchmark/nlu_benchmark/cli/run.py index 00da1a2..9141611 100644 --- a/benchmark/nlu_benchmark/cli/run.py +++ b/benchmark/nlu_benchmark/cli/run.py @@ -102,6 +102,7 @@ def run(data_file_path, data_file_type='yaml', bothub_user_token=None, ## Test + wit_matched_entities: int = 0 test_bothub_result = [] test_wit_result = [] result_intent_test = '' @@ -117,11 +118,9 @@ def analyze_wrapper(text, expected={}): analyze = analyze_response.json() analyze_answer = analyze.get('answer') analyze_intent = analyze_answer.get('intent') - result_intent_test = [] + result_intent_test = 'FALSE' if expected[0].get('intent') == analyze_intent.get('name'): - result_intent_test = 'OK' - else: - result_intent_test = 'FAILURE' + result_intent_test = 'OK' logger.info('Bothub return:') logger.info(f' - intent: {analyze_intent.get("name", "[not detected]")} ' + f'({int(analyze_intent.get("confidence", 0) * 100)}%)') @@ -135,26 +134,53 @@ def analyze_wrapper(text, expected={}): analyze_wit_response = wit.analyze(text,data.get('train')[3],data.get('train')[2].get('wit_token')) entities = analyze_wit_response.json().get('entities') + + wit_entities_percentage:int = 0 + wit_entity_result = '' + entity_matches: int = 0 + known_entities_test: int = 0 + entities_from_wit: int = 0 + intent_result = '' + intent_confidence_result: int = 0 + detected_entities_wit = '|' + intent_test_result = 'FALSE' + for entity in entities: + if entity != 'intent': + for item in entities.get(entity): + entities_from_wit += 1 + detected_entities_wit += '{0}=>{1}|'.format(item.get('value'), entity) + if len(expected) > 1: + for entity in expected[1].get('entities'): + known_entities_test += 1 + if entity.get('entity') == entity: + entity_matches += 1 + # print(str(entity_matches) +'----'+str(entities_from_wit)+'----'+str(known_entities_test)) + if entities_from_wit == known_entities_test == entity_matches: + wit_entity_result = 'OK' + wit_entities_percentage = 100 + elif entities_from_wit != known_entities_test and entity_matches > 0: + wit_entities_percentage = (bothub_entities_count/known_entities_count)*100 + wit_entity_result = 'PARCIAL' + else: + wit_entities_percentage = 0 + wit_entity_result = 'FAILURE' - intent_result = '-' - entities= '|' - # for key in entities: - # print(key) - # entities += str(key)+'|' - # print(entities) if 'intent' in entities: - print('--------------------------Open') - for entity in entities.get('intent'): - # print(entities) - # print('-'+str(entity) +''+entity.get('value')) - test_wit_result.append({ - 'text': text, - 'intent': entity.get('value'), - 'confidence': entity.get('confidence'), - 'expected':expected - }) - print('--------------------------Close') - + intent_result = entities.get('intent')[0].get('value') + intent_confidence_result = entities.get('intent')[0].get('confidence') + if expected[0].get('intent') == intent_result: + intent_test_result = 'OK' + test_wit_result.append({ + 'text': text, + 'intent': intent_result, + 'confidence': intent_confidence_result, + 'expected':expected, + 'entities': detected_entities_wit, + 'result': intent_test_result, + 'entity_percentage': wit_entities_percentage, + 'entity_result': wit_entity_result + }) + intent_result = '' if type_tests: logger.warning('Typing mode ON, press CTRL + C to exit') @@ -179,6 +205,7 @@ def analyze_wrapper(text, expected={}): logger.info(f'Writing CSV files...') csv_headers = "Phrases,Expected intents,Bothub predicts,Confidence accuracy,Result by Intent,Detected entities,Expected entities,Result by Entity, Entity accuracy\n" + expected_entities = '' with open('Bothub_output.csv', 'w') as csv_file: bothub_hits: int = 0 bothub_failures: int = 0 @@ -190,9 +217,8 @@ def analyze_wrapper(text, expected={}): csv_file.write(csv_headers) for example in test_bothub_result: entities = '' - expected_entities = '' + expected_entities = '|' if len(example.get('expected')) > 1: - expected_entities = '|' for entity in example.get('expected')[1].get('entities'): known_entities_count += 1 expected_entities += '{0}=>{1}|'.format(entity.get('value'), entity.get('entity')) @@ -203,8 +229,10 @@ def analyze_wrapper(text, expected={}): entities += '{0}=>{1}|'.format(entity.get('value'), entity.get('entity')) if len(example.get('expected')) > 1: for known_entity in example.get('expected')[1].get('entities'): - if entity.get('entity') == known_entity.get('entity'): + # print(entity.get('entity') + '--'+known_entity.get('entity')) + if entity.get('entity').lower() == known_entity.get('entity').lower(): matched_entities += 1 + print(str(bothub_entities_count)+'---'+str(known_entities_count)+expected_entities+'---'+str(matched_entities)) if example.get('result') == 'OK': bothub_hits += 1 else: @@ -229,12 +257,21 @@ def analyze_wrapper(text, expected={}): csv_file.write('\n' + 'Analized phrases: {0}\nSuccess average: {1}%\nWrong predictions: {2}'.format(len(test_bothub_result), bothub_hits,bothub_failures)) logger.info(f'Bothub_output.csv saved!') + csv_headers = "Phrases,Expected intents,Wit predicts,Confidence accuracy,Result by Intent,Detected entities,Expected entities,Result by Entity, Entity accuracy\n" with open('Wit_output.csv','w') as csv_file: wit_hits: int = 0 wit_failures: int = 0 csv_file.write(csv_headers) for example in test_wit_result: - csv_file.write('{0},{1},{2},{3}%,{4},{5}'.format(example.get('text'),example.get('expected')[0].get('intent'),example.get('intent'),int(example.get('confidence')*100),example.get('result'),entities)) + if len(example.get('expected')) > 1: + expected_entities = '|' + for entity in example.get('expected')[1].get('entities'): + expected_entities += '{0}=>{1}|'.format(entity.get('value'), entity.get('entity')) + if example.get('result') == 'OK': + wit_hits += 1 + else: + wit_failures += 1 + csv_file.write('{0},{1},{2},{3}%,{4},{5},{6},{7},{8}%'.format(example.get('text'),example.get('expected')[0].get('intent'),example.get('intent'),int(example.get('confidence')*100),example.get('result'),example.get('entities'),expected_entities,example.get('entity_result'),example.get('entity_percentage'))) csv_file.write('\n') csv_file.write('\n' + 'Analized phrases: {0}\nSuccess average: {1}%\nWrong predictions: {2}'.format(len(test_wit_result), wit_hits,wit_failures)) logger.info(f'Wit_output.csv saved!') From d26c5a1145163c389d0f248498de033b58ebff6e Mon Sep 17 00:00:00 2001 From: periclesJ Date: Fri, 31 Aug 2018 12:09:54 -0300 Subject: [PATCH 22/23] Finished report generator --- benchmark/datafiles/train_config.yaml | 10 +++++---- benchmark/nlu_benchmark/cli/run.py | 32 ++++++++++++++------------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/benchmark/datafiles/train_config.yaml b/benchmark/datafiles/train_config.yaml index 1d22937..3d78d7b 100644 --- a/benchmark/datafiles/train_config.yaml +++ b/benchmark/datafiles/train_config.yaml @@ -1,9 +1,9 @@ language: pt_br train: - - filepath: /Users/periclesjr/Documents/Benchmark/bothub/benchmark/datafiles/Benchmark_Training.zip - - filename: 'Benchmark_Training' - - wit_token: 'Bearer OWL5XSVZVJNJS4BAVLVXV6STDU7SMPUF' - - wit_version: 20180808 + - filepath: ########## + - filename: ########## + - wit_token: ######### + - wit_version: ####### - filetype: zip tests: - text: O Real Madrid jogará hoje? @@ -389,6 +389,8 @@ tests: entity: modelo_moto - value: honda entity: marca + - value: moto + entity: veiculo - text: Quando sai o novo lançamento do FIAT expected: - intent: automoveis diff --git a/benchmark/nlu_benchmark/cli/run.py b/benchmark/nlu_benchmark/cli/run.py index 9141611..857a4bf 100644 --- a/benchmark/nlu_benchmark/cli/run.py +++ b/benchmark/nlu_benchmark/cli/run.py @@ -138,28 +138,32 @@ def analyze_wrapper(text, expected={}): wit_entities_percentage:int = 0 wit_entity_result = '' entity_matches: int = 0 - known_entities_test: int = 0 - entities_from_wit: int = 0 + known_entities_test = [] + entities_from_wit = [] intent_result = '' intent_confidence_result: int = 0 detected_entities_wit = '|' intent_test_result = 'FALSE' + for entity in entities: if entity != 'intent': for item in entities.get(entity): - entities_from_wit += 1 + entities_from_wit.append(entity.lower()) detected_entities_wit += '{0}=>{1}|'.format(item.get('value'), entity) - if len(expected) > 1: - for entity in expected[1].get('entities'): - known_entities_test += 1 - if entity.get('entity') == entity: - entity_matches += 1 - # print(str(entity_matches) +'----'+str(entities_from_wit)+'----'+str(known_entities_test)) - if entities_from_wit == known_entities_test == entity_matches: + for item in expected: + if 'entities' in item: + for entity in item.get('entities'): + known_entities_test.append(entity.get('entity').lower()) + for item in entities_from_wit: + for expected_item in known_entities_test: + if item == expected_item: + entity_matches += 1 + break + if len(entities_from_wit) == len(known_entities_test) == entity_matches: wit_entity_result = 'OK' wit_entities_percentage = 100 - elif entities_from_wit != known_entities_test and entity_matches > 0: - wit_entities_percentage = (bothub_entities_count/known_entities_count)*100 + elif (len(entities_from_wit) != entity_matches or entity_matches != len(known_entities_test)) and entity_matches > 0: + wit_entities_percentage = (entity_matches/len(known_entities_test))*100 wit_entity_result = 'PARCIAL' else: wit_entities_percentage = 0 @@ -229,10 +233,8 @@ def analyze_wrapper(text, expected={}): entities += '{0}=>{1}|'.format(entity.get('value'), entity.get('entity')) if len(example.get('expected')) > 1: for known_entity in example.get('expected')[1].get('entities'): - # print(entity.get('entity') + '--'+known_entity.get('entity')) if entity.get('entity').lower() == known_entity.get('entity').lower(): matched_entities += 1 - print(str(bothub_entities_count)+'---'+str(known_entities_count)+expected_entities+'---'+str(matched_entities)) if example.get('result') == 'OK': bothub_hits += 1 else: @@ -263,8 +265,8 @@ def analyze_wrapper(text, expected={}): wit_failures: int = 0 csv_file.write(csv_headers) for example in test_wit_result: + expected_entities = '|' if len(example.get('expected')) > 1: - expected_entities = '|' for entity in example.get('expected')[1].get('entities'): expected_entities += '{0}=>{1}|'.format(entity.get('value'), entity.get('entity')) if example.get('result') == 'OK': From faa6ae7bdb9eec58ab85e2547ca3a38369a5cd69 Mon Sep 17 00:00:00 2001 From: periclesJ Date: Tue, 4 Sep 2018 09:46:57 -0300 Subject: [PATCH 23/23] Working on the new bothub version --- benchmark/nlu_benchmark/cli/run.py | 223 ++++++++++++--------- benchmark/nlu_benchmark/services/bothub.py | 2 +- 2 files changed, 131 insertions(+), 94 deletions(-) diff --git a/benchmark/nlu_benchmark/cli/run.py b/benchmark/nlu_benchmark/cli/run.py index 857a4bf..2448a64 100644 --- a/benchmark/nlu_benchmark/cli/run.py +++ b/benchmark/nlu_benchmark/cli/run.py @@ -3,6 +3,7 @@ import json import requests import zipfile +import unicodedata from tqdm import tqdm @@ -73,11 +74,13 @@ def run(data_file_path, data_file_type='yaml', bothub_user_token=None, logger.info('Examples submitted!') else: with zipfile.ZipFile(data.get('train')[0].get('filepath'),'r') as training_file: - with training_file.open('Benchmark_Training/expressions.json') as json_file: + with training_file.open('ProjetoJá/expressions.json') as json_file: training_data = json.loads(json_file.read().decode('utf-8')) for expression in training_data['data']: text = expression['text'] - wit_entities = expression['entities'] + wit_entities = [] + if 'entities' in expression: + wit_entities = expression['entities'] filtered_entities = [] intent = '' for entity in wit_entities: @@ -85,24 +88,37 @@ def run(data_file_path, data_file_type='yaml', bothub_user_token=None, intent = entity['value'].strip('\"').lower() else: new_entity = {} - new_entity['entity'] = entity['entity'] - new_entity['start'] = entity['start'] - new_entity['end'] = entity['end'] - filtered_entities.append(new_entity) - example_submit_response = bothub.submit_from_wit_format(temporary_repository.get('uuid'),[text,filtered_entities,intent]) - logger.debug(f'example trained {example_submit_response.text}') - example_submit_response.raise_for_status() + new_entity['entity'] = ''.join(c for c in unicodedata.normalize('NFD', entity['value'].strip('\"').replace(' ','_').lower()) if unicodedata.category(c) != 'Mn') + + if 'start' in entity: + new_entity['start'] = entity['start'] + else: + new_entity['start'] = 0 + + if 'end' in entity: + new_entity['end'] = entity['end'] + else: + new_entity['end'] = len(text) + + new_entity['label'] = entity['entity'] + + if bool(new_entity): + filtered_entities.append(new_entity) + + if len(filtered_entities) > 0 or intent != '': + example_submit_response = bothub.submit_from_wit_format(temporary_repository.get('uuid'),[text,filtered_entities,intent]) + logger.debug(f'example trained {example_submit_response.text}') + example_submit_response.raise_for_status() logger.info('Examples submitted!') logger.info('Training...') train_response = bothub.train(temporary_repository.get('owner__nickname'), temporary_repository.get('slug')) logger.debug(f'repository train response {train_response.text}') train_response.raise_for_status() - logger.info('Repository trained') + logger.info('Repository trained') ## Test - wit_matched_entities: int = 0 test_bothub_result = [] test_wit_result = [] result_intent_test = '' @@ -116,8 +132,7 @@ def analyze_wrapper(text, expected={}): logger.debug(f'analyze response {analyze_response.text}') analyze_response.raise_for_status() analyze = analyze_response.json() - analyze_answer = analyze.get('answer') - analyze_intent = analyze_answer.get('intent') + analyze_intent = analyze.get('intent') result_intent_test = 'FALSE' if expected[0].get('intent') == analyze_intent.get('name'): result_intent_test = 'OK' @@ -127,7 +142,7 @@ def analyze_wrapper(text, expected={}): test_bothub_result.append({ 'text': text, 'intent': analyze_intent, - 'entities': analyze_answer.get('entities'), + 'entities': analyze.get('entities'), 'expected': expected, 'result': result_intent_test }) @@ -135,56 +150,56 @@ def analyze_wrapper(text, expected={}): analyze_wit_response = wit.analyze(text,data.get('train')[3],data.get('train')[2].get('wit_token')) entities = analyze_wit_response.json().get('entities') - wit_entities_percentage:int = 0 - wit_entity_result = '' - entity_matches: int = 0 - known_entities_test = [] - entities_from_wit = [] - intent_result = '' - intent_confidence_result: int = 0 - detected_entities_wit = '|' - intent_test_result = 'FALSE' + # wit_entities_percentage:int = 0 + # wit_entity_result = '' + # entity_matches: int = 0 + # known_entities_test = [] + # entities_from_wit = [] + # intent_result = '' + # intent_confidence_result: int = 0 + # detected_entities_wit = '|' + # intent_test_result = 'FALSE' - for entity in entities: - if entity != 'intent': - for item in entities.get(entity): - entities_from_wit.append(entity.lower()) - detected_entities_wit += '{0}=>{1}|'.format(item.get('value'), entity) - for item in expected: - if 'entities' in item: - for entity in item.get('entities'): - known_entities_test.append(entity.get('entity').lower()) - for item in entities_from_wit: - for expected_item in known_entities_test: - if item == expected_item: - entity_matches += 1 - break - if len(entities_from_wit) == len(known_entities_test) == entity_matches: - wit_entity_result = 'OK' - wit_entities_percentage = 100 - elif (len(entities_from_wit) != entity_matches or entity_matches != len(known_entities_test)) and entity_matches > 0: - wit_entities_percentage = (entity_matches/len(known_entities_test))*100 - wit_entity_result = 'PARCIAL' - else: - wit_entities_percentage = 0 - wit_entity_result = 'FAILURE' + # for entity in entities: + # if entity != 'intent': + # for item in entities.get(entity): + # entities_from_wit.append(entity.lower()) + # detected_entities_wit += '{0}=>{1}|'.format(item.get('value'), entity) + # for item in expected: + # if 'entities' in item: + # for entity in item.get('entities'): + # known_entities_test.append(entity.get('entity').lower()) + # for item in entities_from_wit: + # for expected_item in known_entities_test: + # if item == expected_item: + # entity_matches += 1 + # break + # if len(entities_from_wit) == len(known_entities_test) == entity_matches: + # wit_entity_result = 'OK' + # wit_entities_percentage = 100 + # elif (len(entities_from_wit) != entity_matches or entity_matches != len(known_entities_test)) and entity_matches > 0: + # wit_entities_percentage = (entity_matches/len(known_entities_test))*100 + # wit_entity_result = 'PARCIAL' + # else: + # wit_entities_percentage = 0 + # wit_entity_result = 'FAILURE' - if 'intent' in entities: - intent_result = entities.get('intent')[0].get('value') - intent_confidence_result = entities.get('intent')[0].get('confidence') - if expected[0].get('intent') == intent_result: - intent_test_result = 'OK' - test_wit_result.append({ - 'text': text, - 'intent': intent_result, - 'confidence': intent_confidence_result, - 'expected':expected, - 'entities': detected_entities_wit, - 'result': intent_test_result, - 'entity_percentage': wit_entities_percentage, - 'entity_result': wit_entity_result - }) - intent_result = '' + # if 'intent' in entities: + # intent_result = entities.get('intent')[0].get('value') + # intent_confidence_result = entities.get('intent')[0].get('confidence') + # if expected[0].get('intent') == intent_result: + # intent_test_result = 'OK' + # test_wit_result.append({ + # 'text': text, + # 'intent': intent_result, + # 'confidence': intent_confidence_result, + # 'expected':expected, + # 'entities': detected_entities_wit, + # 'result': intent_test_result, + # 'entity_percentage': wit_entities_percentage, + # 'entity_result': wit_entity_result + # }) + # intent_result = '' if type_tests: logger.warning('Typing mode ON, press CTRL + C to exit') @@ -226,15 +241,19 @@ def analyze_wrapper(text, expected={}): for entity in example.get('expected')[1].get('entities'): known_entities_count += 1 expected_entities += '{0}=>{1}|'.format(entity.get('value'), entity.get('entity')) - if len(example.get('entities')) > 0: + print(example) + if ('entities' in example) and (len(example.get('entities')) > 0): entities = '|' for entity in example.get('entities'): bothub_entities_count += 1 - entities += '{0}=>{1}|'.format(entity.get('value'), entity.get('entity')) + if (entity == 'other') and len(example.get('entities').get('other')) > 0: + for entity in example.get('entities').get('other'): + entities += '{0}=>{1}|'.format(entity.get('value'), entity.get('entity')) if len(example.get('expected')) > 1: for known_entity in example.get('expected')[1].get('entities'): - if entity.get('entity').lower() == known_entity.get('entity').lower(): - matched_entities += 1 + for entity_without_label in example.get('entities').get('other'): + if entity_without_label.get('entity').lower() == known_entity.get('entity').lower(): + matched_entities += 1 if example.get('result') == 'OK': bothub_hits += 1 else: @@ -259,34 +278,52 @@ def analyze_wrapper(text, expected={}): csv_file.write('\n' + 'Analized phrases: {0}\nSuccess average: {1}%\nWrong predictions: {2}'.format(len(test_bothub_result), bothub_hits,bothub_failures)) logger.info(f'Bothub_output.csv saved!') - csv_headers = "Phrases,Expected intents,Wit predicts,Confidence accuracy,Result by Intent,Detected entities,Expected entities,Result by Entity, Entity accuracy\n" - with open('Wit_output.csv','w') as csv_file: - wit_hits: int = 0 - wit_failures: int = 0 - csv_file.write(csv_headers) - for example in test_wit_result: - expected_entities = '|' - if len(example.get('expected')) > 1: - for entity in example.get('expected')[1].get('entities'): - expected_entities += '{0}=>{1}|'.format(entity.get('value'), entity.get('entity')) - if example.get('result') == 'OK': - wit_hits += 1 - else: - wit_failures += 1 - csv_file.write('{0},{1},{2},{3}%,{4},{5},{6},{7},{8}%'.format(example.get('text'),example.get('expected')[0].get('intent'),example.get('intent'),int(example.get('confidence')*100),example.get('result'),example.get('entities'),expected_entities,example.get('entity_result'),example.get('entity_percentage'))) - csv_file.write('\n') - csv_file.write('\n' + 'Analized phrases: {0}\nSuccess average: {1}%\nWrong predictions: {2}'.format(len(test_wit_result), wit_hits,wit_failures)) - logger.info(f'Wit_output.csv saved!') + # csv_headers = "Phrases,Expected intents,Wit predicts,Confidence accuracy,Result by Intent,Detected entities,Expected entities,Result by Entity, Entity accuracy\n" + # with open('Wit_output.csv','w') as csv_file: + # wit_hits: int = 0 + # wit_failures: int = 0 + # csv_file.write(csv_headers) + # for example in test_wit_result: + # expected_entities = '|' + # if len(example.get('expected')) > 1: + # for entity in example.get('expected')[1].get('entities'): + # expected_entities += '{0}=>{1}|'.format(entity.get('value'), entity.get('entity')) + # if example.get('result') == 'OK': + # wit_hits += 1 + # else: + # wit_failures += 1 + # csv_file.write('{0},{1},{2},{3}%,{4},{5},{6},{7},{8}%'.format(example.get('text'),example.get('expected')[0].get('intent'),example.get('intent'),int(example.get('confidence')*100),example.get('result'),example.get('entities'),expected_entities,example.get('entity_result'),example.get('entity_percentage'))) + # csv_file.write('\n') + # csv_file.write('\n' + 'Analized phrases: {0}\nSuccess average: {1}%\nWrong predictions: {2}'.format(len(test_wit_result), wit_hits,wit_failures)) + # logger.info(f'Wit_output.csv saved!') ## Delete a temporary repository - delete_repository_response = bothub.delete_repository( - temporary_repository.get('owner__nickname'), - temporary_repository.get('slug')) + # delete_repository_response = bothub.delete_repository( + # temporary_repository.get('owner__nickname'), + # temporary_repository.get('slug')) + + # try: + # delete_repository_response.raise_for_status() + # except Exception as e: + # logger.warning('Bothub temporary repository not deleted, manually delete') + # logger.debug(delete_repository_response.text) - try: - delete_repository_response.raise_for_status() - except Exception as e: - logger.warning('Bothub temporary repository not deleted, manually delete') - logger.debug(delete_repository_response.text) + logger.info(f'Bothub temporary repository deleted') - logger.info(f'Bothub temporary repository deleted') \ No newline at end of file + bothub.delete_repository(temporary_repository.get('owner__nickname'),'577O11KN2BS06MXG') + bothub.delete_repository(temporary_repository.get('owner__nickname'),'L6FLKN2GQHAJ0M80') + bothub.delete_repository(temporary_repository.get('owner__nickname'),'DHIHOM1XEQ4BAM95') + bothub.delete_repository(temporary_repository.get('owner__nickname'),'PEB90GJREPHJCL1U') + bothub.delete_repository(temporary_repository.get('owner__nickname'),'GT4GFLO03CZHHSHF') + bothub.delete_repository(temporary_repository.get('owner__nickname'),'28QTVA36R17TQJUA') + bothub.delete_repository(temporary_repository.get('owner__nickname'),'14LR7RNRFH3JCBJB') + bothub.delete_repository(temporary_repository.get('owner__nickname'),'JU5TMUUMRQM26F0D') + bothub.delete_repository(temporary_repository.get('owner__nickname'),'UTIJPUJHV4H2VEJT') + bothub.delete_repository(temporary_repository.get('owner__nickname'),'WA1JC2PP77EYXSWS') + bothub.delete_repository(temporary_repository.get('owner__nickname'),'BDLOYC6SFI256GU0') + bothub.delete_repository(temporary_repository.get('owner__nickname'),'HP1572M3AOP9D562') + bothub.delete_repository(temporary_repository.get('owner__nickname'),'2JERLTQIZM14J2N9') + bothub.delete_repository(temporary_repository.get('owner__nickname'),'7YWPENL495FBP5SO') + bothub.delete_repository(temporary_repository.get('owner__nickname'),'DNFPW93RCWR4IF6L') + bothub.delete_repository(temporary_repository.get('owner__nickname'),'QANTJ2C8W3948AA4') + bothub.delete_repository(temporary_repository.get('owner__nickname'),'HHSUD0RXCH72P5RY') \ No newline at end of file diff --git a/benchmark/nlu_benchmark/services/bothub.py b/benchmark/nlu_benchmark/services/bothub.py index d2d9956..3a7f5bb 100644 --- a/benchmark/nlu_benchmark/services/bothub.py +++ b/benchmark/nlu_benchmark/services/bothub.py @@ -23,7 +23,7 @@ def api_request(cls, path, data=None, headers={}, user_token=None, headers.update({ 'Content-Type': 'application/json' }) request = Request( method, - f'https://bothub.it/api/{path}/', + f'http://staging.bothub.it:8000/{path}/', data=json.dumps(data), headers=headers) prepped = request.prepare()