diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ea1e99dc2..a3f7f304a 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.9.11 +current_version = 1.10.1 commit = True tag = True @@ -21,8 +21,6 @@ replace = {new_version}' [bumpversion:file:docassemble_webapp/docassemble/webapp/__init__.py] -[bumpversion:file:docassemble_webapp/docassemble/webapp/setup.py] - [bumpversion:file:Docker/VERSION] [bumpversion:file:docassemble_webapp/docassemble/webapp/data/VERSION.txt] diff --git a/.gitignore b/.gitignore index 8e25131b7..ad549198f 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ audit.py conflict_check.yml publish.sh pypi-publish.sh +pypi-publish-dev.sh newversion.sh json_files posttoslack.sh @@ -65,10 +66,16 @@ settings.json .dir-locals.el .flake8 mypy.ini +docassemble_base/mypy.ini +docassemble_demo/mypy.ini +docassemble_webapp/mypy.ini stylelint.config.js .stylelintrc.json .sass-cache/ CLAUDE.md .claude/ .claudeignore -__pycache__ \ No newline at end of file +__pycache__ +pyproject.toml +autoimport.py +runpylint.py \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 9db31e848..7da232365 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,57 @@ # Change Log +## [1.10.1] - 2026-06-29 + +### Fixed + +- Errors related to add-on package installation. + +## [1.10.0] - 2026-06-28 + +### Added + +- The `enable api`, `enable email server`, `enable object storage`, + `enable daglobal`, `enable json storage`, `enable sms interface`, + `enable faxing`, and `enable tts` Configuration directives. +- The `pip trusted host` Configuration directive. + +### Changed + +- The internal organization of the source code has changed + significantly. Documented functionality has not changed, but if your + code imports undocumented names, you may find that the name has + moved to another module. Testing your interviews on a development + server is recommended before upgrading a production server. +- `pluggy` is now used to give `docassemble.base` access to + functionality in `docassemble.webapp`. +- Flask blueprints are now used for features of + `docassemble.webapp`. Configuration directives like `enable + training` and `allow log viewing` will now prevent the importing of + the code underlying the disabled features. Turning off these + features can decrease uWSGI memory usage by 29%. +- SQLAlchemy models now inherit from `sqlalchemy.org.DeclarativeBase` + rather than `db.model`. +- Celery tasks are now invoked using signatures. +- Added a dependency on `pyopenssl` not because it is needed, but + because a version conflict with `cryptography` will prevent the + web application from starting. + +### Fixed + +- Issue with `list collect` and `input type: hidden`. + +## [1.9.13] - 2026-06-03 + +### Fixed + +- Issue with converting fillable PDFs to PDF/A. + +## [1.9.12] - 2026-05-11 + +### Fixed + +- Issue with `Flask.SocketIO`. + ## [1.9.11] - 2026-05-08 ### Fixed diff --git a/Docker/VERSION b/Docker/VERSION index 5e9287b86..4dae2985b 100644 --- a/Docker/VERSION +++ b/Docker/VERSION @@ -1 +1 @@ -1.9.11 +1.10.1 diff --git a/Docker/config/config.yml.dist b/Docker/config/config.yml.dist index ac41fa1a1..1386874c8 100644 --- a/Docker/config/config.yml.dist +++ b/Docker/config/config.yml.dist @@ -149,3 +149,13 @@ gotenberg: password: {{GOTENBERGPASSWORD}} javascript defer: True use nginx to serve files: {{USENGINXTOSERVEFILES}} +enable api: True +enable daglobal: True +enable email server: True +enable faxing: True +enable json storage: True +enable monitor: True +enable object storage: True +enable sms interface: True +enable training: True +enable tts: True diff --git a/Docker/cron/docassemble-cron-daily.sh b/Docker/cron/docassemble-cron-daily.sh index 9c598969a..3fb6a53d3 100755 --- a/Docker/cron/docassemble-cron-daily.sh +++ b/Docker/cron/docassemble-cron-daily.sh @@ -42,15 +42,17 @@ if [[ $CONTAINERROLE =~ .*:(all):.* ]] && [ "${USEHTTPS:-false}" == "true" ] && cp /tmp/letsencrypt.tar.gz "${DA_ROOT}/backup/letsencrypt.tar.gz" fi rm -f /tmp/letsencrypt.tar.gz - if [ ! -f /etc/ssl/docassemble/exim.crt ] && [ ! -f /etc/ssl/docassemble/exim.key ]; then - cp "/etc/letsencrypt/live/${DAHOSTNAME}/fullchain.pem" /etc/exim4/exim.crt - cp "/etc/letsencrypt/live/${DAHOSTNAME}/privkey.pem" /etc/exim4/exim.key - chown root:Debian-exim /etc/exim4/exim.crt - chown root:Debian-exim /etc/exim4/exim.key - chmod 640 /etc/exim4/exim.crt - chmod 640 /etc/exim4/exim.key - supervisorctl ${DASUPERVISOROPTS}--serverurl http://localhost:9001 stop exim4 - supervisorctl ${DASUPERVISOROPTS}--serverurl http://localhost:9001 start exim4 + if [ "$ENABLEEMAILSERVER" == "true" ]; then + if [ ! -f /etc/ssl/docassemble/exim.crt ] && [ ! -f /etc/ssl/docassemble/exim.key ]; then + cp "/etc/letsencrypt/live/${DAHOSTNAME}/fullchain.pem" /etc/exim4/exim.crt + cp "/etc/letsencrypt/live/${DAHOSTNAME}/privkey.pem" /etc/exim4/exim.key + chown root:Debian-exim /etc/exim4/exim.crt + chown root:Debian-exim /etc/exim4/exim.key + chmod 640 /etc/exim4/exim.crt + chmod 640 /etc/exim4/exim.key + supervisorctl ${DASUPERVISOROPTS}--serverurl http://localhost:9001 stop exim4 + supervisorctl ${DASUPERVISOROPTS}--serverurl http://localhost:9001 start exim4 + fi fi fi diff --git a/Docker/initialize.sh b/Docker/initialize.sh index 49d170fad..46ac6e8b4 100755 --- a/Docker/initialize.sh +++ b/Docker/initialize.sh @@ -987,6 +987,14 @@ else su -c "source \"${DA_ACTIVATE}\" && pip config unset global.extra-index-url" www-data &> /dev/null fi + echo "initialize: Checking to see if pip trusted host is used" >&2 + + if [ "${PIPTRUSTEDHOST:-null}" != "null" ]; then + su -c "source \"${DA_ACTIVATE}\" && pip config set global.trusted-host \"${PIPTRUSTEDHOST}\"" www-data + else + su -c "source \"${DA_ACTIVATE}\" && pip config set global.trusted-host github.com" www-data &> /dev/null + fi + if [ -n "$PYTHONPACKAGES" ]; then echo "initialize: Installing Python packages specified in the Configuration" >&2 for PACKAGE in "${PYTHONPACKAGES[@]}"; do @@ -1018,6 +1026,14 @@ else pip config unset global.extra-index-url &> /dev/null fi + echo "initialize: Checking to see if pip trusted host is used" >&2 + + if [ "${PIPTRUSTEDHOST:-null}" != "null" ]; then + pip config set global.trusted-host "${PIPTRUSTEDHOST}" + else + pip config set global.trusted-host "github.com" &> /dev/null + fi + if [ -n "$PYTHONPACKAGES" ]; then echo "initialize: Installing Python packages specified in the Configuration" >&2 for PACKAGE in "${PYTHONPACKAGES[@]}"; do @@ -1339,8 +1355,10 @@ if [ "${DAWEBSERVER:-nginx}" = "none" ]; then ${SUPERVISORCMD} stop nascent &> /dev/null NASCENTRUNNING=false; if [[ $CONTAINERROLE =~ .*:(all|web):.* ]]; then - echo "initialize: Starting websockets" >&2 - ${SUPERVISORCMD} start websockets + if [ "${ENABLEMONITOR:-true}" = "true" ]; then + echo "initialize: Starting websockets" >&2 + ${SUPERVISORCMD} start websockets + fi echo "initialize: Starting uwsgi" >&2 ${SUPERVISORCMD} start uwsgi fi @@ -1427,8 +1445,10 @@ if [ "${DAWEBSERVER:-nginx}" = "nginx" ]; then fi fi if [[ $CONTAINERROLE =~ .*:(all|web):.* ]]; then - echo "initialize: Starting websockets" >&2 - ${SUPERVISORCMD} start websockets + if [ "${ENABLEMONITOR:-true}" = "true" ]; then + echo "initialize: Starting websockets" >&2 + ${SUPERVISORCMD} start websockets + fi echo "initialize: Starting uwsgi" >&2 ${SUPERVISORCMD} start uwsgi fi @@ -1575,7 +1595,7 @@ if [ "${DAWEBSERVER:-nginx}" = "apache" ]; then a2ensite docassemble-log fi - if [[ $CONTAINERROLE =~ .*:(all|web):.* ]]; then + if [[ $CONTAINERROLE =~ .*:(all|web):.* ]] && [ "${ENABLEMONITOR:-true}" = "true" ]; then echo "initialize: Starting websockets" >&2 ${SUPERVISORCMD} start websockets fi @@ -1628,81 +1648,83 @@ if [ "$CRONRUNNING" == "false" ]; then ${SUPERVISORCMD} start cron fi -if exiwhat 2> /dev/null | grep -q listening; then - EXIM4RUNNING=true -else - EXIM4RUNNING=false -fi +if [ "${ENABLEEMAILSERVER:-true}" == "true" ]; then + if exiwhat 2> /dev/null | grep -q listening; then + EXIM4RUNNING=true + else + EXIM4RUNNING=false + fi -if [ "$EXIM4RUNNING" == "false" ] && [[ $CONTAINERROLE =~ .*:(all|mail):.* && ($DBTYPE = "postgresql" || $DBTYPE = "mysql") ]]; then - echo "initialize: Starting exim4" >&2 - if [ "${DAREADONLYFILESYSTEM:-false}" == "false" ]; then - if [ -f /usr/share/docassemble/config/exim4-update ] && [ "${DAHOSTNAME}" != "localhost" ]; then - sed "s/dc_other_hostnames='\**'/dc_other_hostnames='${DAHOSTNAME}'/" /usr/share/docassemble/config/exim4-update > /tmp/temp-exim4-update - if [ ! -f /etc/exim4/update-exim4.conf.conf ] || ! cmp -s /tmp/temp-exim4-update /etc/exim4/update-exim4.conf.conf; then - cp /tmp/temp-exim4-update /etc/exim4/update-exim4.conf.conf - update-exim4.conf - fi - rm -f /tmp/temp-exim4-update - fi - rm -f /etc/cron.daily/exim4-base - ln -s /usr/share/docassemble/cron/exim4-base /etc/cron.daily/exim4-base - if [ "${DBTYPE}" = "postgresql" ]; then - cp "${DA_ROOT}/config/exim4-router-postgresql" /etc/exim4/dbrouter - if [ "${DBHOST:-null}" != "null" ]; then - echo -n 'hide pgsql_servers = '${DBHOST} > /etc/exim4/dbinfo - else - echo -n 'hide pgsql_servers = localhost' > /etc/exim4/dbinfo - fi - if [ "${DBPORT:-null}" != "null" ]; then - echo -n '::'${DBPORT} >> /etc/exim4/dbinfo - fi - echo '/'${DBNAME}'/'${DBUSER}'/'${DBPASSWORD} >> /etc/exim4/dbinfo - fi - if [ "$DBTYPE" = "mysql" ]; then - cp "${DA_ROOT}/config/exim4-router-mysql" /etc/exim4/dbrouter - if [ "${DBHOST:-null}" != "null" ]; then - echo -n 'hide mysql_servers = '${DBHOST} > /etc/exim4/dbinfo - else - echo -n 'hide mysql_servers = localhost' > /etc/exim4/dbinfo - fi - if [ "${DBPORT:-null}" != "null" ]; then - echo -n '::'${DBPORT} >> /etc/exim4/dbinfo - fi - echo '/'${DBNAME}'/'${DBUSER}'/'${DBPASSWORD} >> /etc/exim4/dbinfo - fi - if [ "${DBTYPE}" = "postgresql" ]; then - echo 'DAQUERY = select short from '${DBTABLEPREFIX}"shortener where short='\${quote_pgsql:\$local_part}'" >> /etc/exim4/dbinfo - fi - if [ "${DBTYPE}" = "mysql" ]; then - echo 'DAQUERY = select short from '${DBTABLEPREFIX}"shortener where short='\${quote_mysql:\$local_part}'" >> /etc/exim4/dbinfo - fi - if [ -f /etc/ssl/docassemble/exim.crt ] && [ -f /etc/ssl/docassemble/exim.key ]; then - cp /etc/ssl/docassemble/exim.crt /etc/exim4/exim.crt - cp /etc/ssl/docassemble/exim.key /etc/exim4/exim.key - chown root:Debian-exim /etc/exim4/exim.crt - chown root:Debian-exim /etc/exim4/exim.key - chmod 640 /etc/exim4/exim.crt - chmod 640 /etc/exim4/exim.key - echo 'MAIN_TLS_ENABLE = yes' >> /etc/exim4/dbinfo - elif [[ $CONTAINERROLE =~ .*:(all|web):.* ]] && [ "${USELETSENCRYPT:-false}" == "true" ] && [ -f "/etc/letsencrypt/live/${DAHOSTNAME}/cert.pem" ] && [ -f "/etc/letsencrypt/live/${DAHOSTNAME}/privkey.pem" ]; then - cp "/etc/letsencrypt/live/${DAHOSTNAME}/fullchain.pem" /etc/exim4/exim.crt - cp "/etc/letsencrypt/live/${DAHOSTNAME}/privkey.pem" /etc/exim4/exim.key - chown root:Debian-exim /etc/exim4/exim.crt - chown root:Debian-exim /etc/exim4/exim.key - chmod 640 /etc/exim4/exim.crt - chmod 640 /etc/exim4/exim.key - echo 'MAIN_TLS_ENABLE = yes' >> /etc/exim4/dbinfo - else - echo 'MAIN_TLS_ENABLE = no' >> /etc/exim4/dbinfo - fi - chmod og-rwx /etc/exim4/dbinfo + if [ "$EXIM4RUNNING" == "false" ] && [[ $CONTAINERROLE =~ .*:(all|mail):.* && ($DBTYPE = "postgresql" || $DBTYPE = "mysql") ]]; then + echo "initialize: Starting exim4" >&2 + if [ "${DAREADONLYFILESYSTEM:-false}" == "false" ]; then + if [ -f /usr/share/docassemble/config/exim4-update ] && [ "${DAHOSTNAME}" != "localhost" ]; then + sed "s/dc_other_hostnames='\**'/dc_other_hostnames='${DAHOSTNAME}'/" /usr/share/docassemble/config/exim4-update > /tmp/temp-exim4-update + if [ ! -f /etc/exim4/update-exim4.conf.conf ] || ! cmp -s /tmp/temp-exim4-update /etc/exim4/update-exim4.conf.conf; then + cp /tmp/temp-exim4-update /etc/exim4/update-exim4.conf.conf + update-exim4.conf + fi + rm -f /tmp/temp-exim4-update + fi + rm -f /etc/cron.daily/exim4-base + ln -s /usr/share/docassemble/cron/exim4-base /etc/cron.daily/exim4-base + if [ "${DBTYPE}" = "postgresql" ]; then + cp "${DA_ROOT}/config/exim4-router-postgresql" /etc/exim4/dbrouter + if [ "${DBHOST:-null}" != "null" ]; then + echo -n 'hide pgsql_servers = '${DBHOST} > /etc/exim4/dbinfo + else + echo -n 'hide pgsql_servers = localhost' > /etc/exim4/dbinfo + fi + if [ "${DBPORT:-null}" != "null" ]; then + echo -n '::'${DBPORT} >> /etc/exim4/dbinfo + fi + echo '/'${DBNAME}'/'${DBUSER}'/'${DBPASSWORD} >> /etc/exim4/dbinfo + fi + if [ "$DBTYPE" = "mysql" ]; then + cp "${DA_ROOT}/config/exim4-router-mysql" /etc/exim4/dbrouter + if [ "${DBHOST:-null}" != "null" ]; then + echo -n 'hide mysql_servers = '${DBHOST} > /etc/exim4/dbinfo + else + echo -n 'hide mysql_servers = localhost' > /etc/exim4/dbinfo + fi + if [ "${DBPORT:-null}" != "null" ]; then + echo -n '::'${DBPORT} >> /etc/exim4/dbinfo + fi + echo '/'${DBNAME}'/'${DBUSER}'/'${DBPASSWORD} >> /etc/exim4/dbinfo + fi + if [ "${DBTYPE}" = "postgresql" ]; then + echo 'DAQUERY = select short from '${DBTABLEPREFIX}"shortener where short='\${quote_pgsql:\$local_part}'" >> /etc/exim4/dbinfo + fi + if [ "${DBTYPE}" = "mysql" ]; then + echo 'DAQUERY = select short from '${DBTABLEPREFIX}"shortener where short='\${quote_mysql:\$local_part}'" >> /etc/exim4/dbinfo + fi + if [ -f /etc/ssl/docassemble/exim.crt ] && [ -f /etc/ssl/docassemble/exim.key ]; then + cp /etc/ssl/docassemble/exim.crt /etc/exim4/exim.crt + cp /etc/ssl/docassemble/exim.key /etc/exim4/exim.key + chown root:Debian-exim /etc/exim4/exim.crt + chown root:Debian-exim /etc/exim4/exim.key + chmod 640 /etc/exim4/exim.crt + chmod 640 /etc/exim4/exim.key + echo 'MAIN_TLS_ENABLE = yes' >> /etc/exim4/dbinfo + elif [[ $CONTAINERROLE =~ .*:(all|web):.* ]] && [ "${USELETSENCRYPT:-false}" == "true" ] && [ -f "/etc/letsencrypt/live/${DAHOSTNAME}/cert.pem" ] && [ -f "/etc/letsencrypt/live/${DAHOSTNAME}/privkey.pem" ]; then + cp "/etc/letsencrypt/live/${DAHOSTNAME}/fullchain.pem" /etc/exim4/exim.crt + cp "/etc/letsencrypt/live/${DAHOSTNAME}/privkey.pem" /etc/exim4/exim.key + chown root:Debian-exim /etc/exim4/exim.crt + chown root:Debian-exim /etc/exim4/exim.key + chmod 640 /etc/exim4/exim.crt + chmod 640 /etc/exim4/exim.key + echo 'MAIN_TLS_ENABLE = yes' >> /etc/exim4/dbinfo + else + echo 'MAIN_TLS_ENABLE = no' >> /etc/exim4/dbinfo + fi + chmod og-rwx /etc/exim4/dbinfo + fi + ${SUPERVISORCMD} start exim4 + elif [ "${DAREADONLYFILESYSTEM:-false}" == "false" ]; then + echo "initialize: Disabling exim4 cron" >&2 + rm -f /etc/cron.daily/exim4-base + ln -s /usr/share/docassemble/cron/donothing /etc/cron.daily/exim4-base fi - ${SUPERVISORCMD} start exim4 -elif [ "${DAREADONLYFILESYSTEM:-false}" == "false" ]; then - echo "initialize: Disabling exim4 cron" >&2 - rm -f /etc/cron.daily/exim4-base - ln -s /usr/share/docassemble/cron/donothing /etc/cron.daily/exim4-base fi if [[ $CONTAINERROLE =~ .*:(log):.* ]] || [ "$OTHERLOGSERVER" == "true" ]; then diff --git a/Docker/pip.conf b/Docker/pip.conf index 17e861060..1ce25130d 100644 --- a/Docker/pip.conf +++ b/Docker/pip.conf @@ -1,5 +1,2 @@ [global] disable-pip-version-check = True -[install] -trusted-host = - github.com diff --git a/Docker/reset.sh b/Docker/reset.sh index 23371e7ca..741c4cefa 100755 --- a/Docker/reset.sh +++ b/Docker/reset.sh @@ -75,6 +75,12 @@ update_pip_config(){ else pip config unset global.extra-index-url &> /dev/null fi + + if [ "${PIPTRUSTEDHOST:-null}" != "null" ]; then + pip config set global.trusted-host "${PIPTRUSTEDHOST}" + else + pip config set global.trusted-host "github.com" &> /dev/null + fi } restart_websockets() { echo "`date` stopping websockets" >&2 @@ -108,7 +114,7 @@ if [ "${DAALLOWUPDATES:-true}" == "true" ]; then update_pip_config & fi -if [[ $CONTAINERROLE =~ .*:(all|web):.* ]]; then +if [[ $CONTAINERROLE =~ .*:(all|web):.* ]] && [ "${ENABLEMONITOR:-true}" = "true" ]; then restart_websockets & fi diff --git a/Docker/restart-post-logrotate.sh b/Docker/restart-post-logrotate.sh index 525e88b91..a478a74c9 100755 --- a/Docker/restart-post-logrotate.sh +++ b/Docker/restart-post-logrotate.sh @@ -39,7 +39,7 @@ if [[ $CONTAINERROLE =~ .*:(all|celery):.* ]]; then ${SUPERVISORCMD} start celerysingle > /dev/null || exit 1 fi -if [[ $CONTAINERROLE =~ .*:(all|web):.* ]]; then +if [[ $CONTAINERROLE =~ .*:(all|web):.* ]] && [ "${ENABLEMONITOR:-true}" = "true" ]; then ${SUPERVISORCMD} stop websockets > /dev/null || exit 1 sleep 1 ${SUPERVISORCMD} start websockets > /dev/null || exit 1 diff --git a/Docker/run-apache.sh b/Docker/run-apache.sh index a652aca24..861f39b28 100755 --- a/Docker/run-apache.sh +++ b/Docker/run-apache.sh @@ -12,7 +12,7 @@ source /dev/stdin < <(su -c "source \"$DA_ACTIVATE\" && python -m docassemble.ba set -- $LOCALE export LANG=$1 -if [[ '$(dpkg --print-architecture)' == 'amd64' ]]; then +if [[ $(dpkg --print-architecture) == 'amd64' ]]; then CURRENTARCH=x86_64 else CURRENTARCH=aarch64 diff --git a/Docker/run-cron.sh b/Docker/run-cron.sh index a8f4e74ca..ffa734456 100755 --- a/Docker/run-cron.sh +++ b/Docker/run-cron.sh @@ -16,4 +16,6 @@ source /dev/stdin < <(su -c "source \"$DA_ACTIVATE\" && python -m docassemble.ba set -- $LOCALE export LANG=$1 -exec nice -n 19 su -c "source \"$DA_ACTIVATE\" && python -m docassemble.webapp.cron \"$DA_CONFIG_FILE\" -type $CRONTYPE" www-data +export IN_CRON=true + +exec nice -n 19 su -c "source \"$DA_ACTIVATE\" && flask --app docassemble.webapp.server cron run $CRONTYPE" www-data diff --git a/Dockerfile b/Dockerfile index 3143b6ae6..25acc97d2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,9 +54,9 @@ bash -c \ && pip install --upgrade pip==26.0.1 \ && pip install --upgrade mod_wsgi==5.0.2 \ && pip install --upgrade \ - certbot==5.2.2 \ - certbot-apache==5.2.2 \ - certbot-nginx==5.2.2 \ + certbot==5.6.0 \ + certbot-apache==5.6.0 \ + certbot-nginx==5.6.0 \ minio==7.2.20 \ uWSGI==2.0.31 \ && pip install \ diff --git a/LICENSE.txt b/LICENSE.txt index f9a98fdcf..a3c07394e 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015-2025 Jonathan Pyle +Copyright (c) 2015-2026 Jonathan Pyle Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/docassemble_base/LICENSE.txt b/docassemble_base/LICENSE.txt index f9a98fdcf..a3c07394e 100644 --- a/docassemble_base/LICENSE.txt +++ b/docassemble_base/LICENSE.txt @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015-2025 Jonathan Pyle +Copyright (c) 2015-2026 Jonathan Pyle Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/docassemble_base/config.yml b/docassemble_base/config.yml index c931d83c6..398af6b3c 100644 --- a/docassemble_base/config.yml +++ b/docassemble_base/config.yml @@ -99,3 +99,13 @@ allow configuration editing: True javascript defer: True use nginx to serve files: True restrict input variables: True +enable api: True +enable daglobal: True +enable email server: True +enable faxing: True +enable json storage: True +enable monitor: True +enable object storage: True +enable sms interface: True +enable training: True +enable tts: True diff --git a/docassemble_base/docassemble/base/DA.py b/docassemble_base/docassemble/base/DA.py index 8ee28270c..dadd365e7 100644 --- a/docassemble_base/docassemble/base/DA.py +++ b/docassemble_base/docassemble/base/DA.py @@ -1,3 +1,4 @@ +# pylint: disable=invalid-name __all__ = [] diff --git a/docassemble_base/docassemble/base/__init__.py b/docassemble_base/docassemble/base/__init__.py index b95b03c16..a0865bba6 100644 --- a/docassemble_base/docassemble/base/__init__.py +++ b/docassemble_base/docassemble/base/__init__.py @@ -1 +1 @@ -__version__ = "1.9.11" +__version__ = "1.10.1" diff --git a/docassemble_base/docassemble/base/amazon.py b/docassemble_base/docassemble/base/amazon.py index c04c0f54b..a796dd44b 100644 --- a/docassemble_base/docassemble/base/amazon.py +++ b/docassemble_base/docassemble/base/amazon.py @@ -7,7 +7,7 @@ epoch = datetime.datetime(1970, 1, 1, 0, 0, tzinfo=datetime.timezone.utc) -class s3object: +class S3Object: def __init__(self, s3_config): self.upload_args = {} @@ -41,23 +41,23 @@ def __init__(self, s3_config): self.bucket_name = s3_config['bucket'] def get_key(self, key_name): - return s3key(self, self.conn.Object(self.bucket_name, key_name)) + return S3Key(self, self.conn.Object(self.bucket_name, key_name)) def search_key(self, key_name): for key in self.bucket.objects.filter(Prefix=key_name, Delimiter='/'): if key.key == key_name: - return s3key(self, self.conn.Object(self.bucket_name, key.key)) + return S3Key(self, self.conn.Object(self.bucket_name, key.key)) return None def list_keys(self, prefix): output = [] for obj in self.bucket.objects.filter(Prefix=prefix): - new_key = s3key(self, obj) + new_key = S3Key(self, obj) output.append(new_key) return output -class s3key: +class S3Key: def __init__(self, s3_object, key_obj): self.s3_object = s3_object diff --git a/docassemble_base/docassemble/base/astparser.py b/docassemble_base/docassemble/base/astparser.py index 02dafcbca..5e1f7f599 100644 --- a/docassemble_base/docassemble/base/astparser.py +++ b/docassemble_base/docassemble/base/astparser.py @@ -1,3 +1,4 @@ +# pylint: disable=invalid-name import ast import re @@ -7,7 +8,7 @@ valid_variable_match = re.compile(r'^[^\d][A-Za-z0-9\_]*$') -class myextract(ast.NodeVisitor): +class MyExtract(ast.NodeVisitor): def __init__(self): self.stack = [] @@ -66,7 +67,7 @@ def visit_Subscript(self, node): self.in_subscript -= 1 -class myvisitnode(ast.NodeVisitor): +class MyVisitNode(ast.NodeVisitor): def __init__(self): self.names = {} @@ -90,7 +91,7 @@ def visit_Call(self, node): def visit_Subscript(self, node): if node not in self.calls: - crawler = myextract() + crawler = MyExtract() crawler.visit(node) if not crawler.seen_complexity: self.names[fix_assign.sub(r'\1', (".".join(reversed(crawler.stack))))] = 1 @@ -98,7 +99,7 @@ def visit_Subscript(self, node): def visit_Attribute(self, node): if node not in self.calls: - crawler = myextract() + crawler = MyExtract() crawler.visit(node) if not crawler.seen_complexity: self.names[fix_assign.sub(r'\1', (".".join(reversed(crawler.stack))))] = 1 @@ -117,11 +118,11 @@ def visit_Assign(self, node): for subnode in val: if isinstance(subnode, ast.Tuple): for subsubnode in subnode.elts: - crawler = myextract() + crawler = MyExtract() crawler.visit(subsubnode) self.targets[fix_assign.sub(r'\1', ".".join(reversed(crawler.stack)))] = 1 else: - crawler = myextract() + crawler = MyExtract() crawler.visit(subnode) self.targets[fix_assign.sub(r'\1', ".".join(reversed(crawler.stack)))] = 1 self.depth += 1 @@ -132,7 +133,7 @@ def visit_Assign(self, node): def visit_AugAssign(self, node): for key, val in ast.iter_fields(node): if key == 'target': - crawler = myextract() + crawler = MyExtract() crawler.visit(val) self.targets[fix_assign.sub(r'\1', ".".join(reversed(crawler.stack)))] = 1 self.depth += 1 @@ -143,7 +144,7 @@ def visit_AugAssign(self, node): def visit_AnnAssign(self, node): for key, val in ast.iter_fields(node): if key == 'target': - crawler = myextract() + crawler = MyExtract() crawler.visit(val) self.targets[fix_assign.sub(r'\1', ".".join(reversed(crawler.stack)))] = 1 self.depth += 1 @@ -241,7 +242,7 @@ def visit_Name(self, node): self.generic_visit(node) -class detectIllegal(ast.NodeVisitor): +class DetectIllegal(ast.NodeVisitor): def __init__(self): self.illegal = False @@ -467,7 +468,7 @@ def visit_Starred(self, node): ast.NodeVisitor.generic_visit(self, node) -class detectIllegalQuery(ast.NodeVisitor): +class DetectIllegalQuery(ast.NodeVisitor): def __init__(self): self.illegal = False diff --git a/docassemble_base/docassemble/base/background.py b/docassemble_base/docassemble/base/background.py new file mode 100644 index 000000000..824724ab2 --- /dev/null +++ b/docassemble_base/docassemble/base/background.py @@ -0,0 +1,63 @@ +from docassemble.base.thread_context import this_thread +from docassemble.base.hooks import get_server_redis, get_celery_app, get_task + + +class BackgroundResult: + + def __init__(self, result): + for attr in ('value', 'error_type', 'error_trace', 'error_message', 'variables'): + if hasattr(result, attr): + setattr(self, attr, getattr(result, attr)) + else: + setattr(self, attr, None) + + +class MyAsyncResult: + + def wait(self): + if not hasattr(self, '_cached_result'): + self._cached_result = BackgroundResult(get_task(self.obj).get()) + return True + + def failed(self): + if not hasattr(self, '_cached_result'): + self._cached_result = BackgroundResult(get_task(self.obj).get()) + if self._cached_result.error_type is not None: + return True + return False + + def ready(self): + return get_task(self.obj).ready() + + def result(self): + if not hasattr(self, '_cached_result'): + self._cached_result = BackgroundResult(get_task(self.obj).get()) + return self._cached_result + + def get(self): + if not hasattr(self, '_cached_result'): + self._cached_result = BackgroundResult(get_task(self.obj).get()) + return self._cached_result.value + + def revoke(self, terminate=True): + return get_task(self.obj).revoke(terminate=terminate) + + def status(self): + return get_task(self.obj).status + + def state(self): + return get_task(self.obj).state + + def date_done(self): + return get_task(self.obj).date_done + + +def bg_action(action, ui_notification, **kwargs): + result = MyAsyncResult() + result.obj = get_celery_app().signature('tasks.background_action', args=[this_thread.current_info['yaml_filename'], this_thread.current_info['user'], this_thread.current_info['session'], this_thread.current_info['secret'], this_thread.current_info['url'], this_thread.current_info['url_root'], {'action': action, 'arguments': kwargs}], kwargs={"extra": ui_notification}).delay() + if ui_notification is not None: + worker_key = 'da:worker:uid:' + str(this_thread.current_info['session']) + ':i:' + str(this_thread.current_info['yaml_filename']) + ':userid:' + str(this_thread.current_info['user']['the_user_id']) + # logmessage("worker_caller: id is " + str(result.obj.id) + " and key is " + worker_key) + get_server_redis().rpush(worker_key, result.obj.id) + # logmessage("worker_caller: id is " + str(result.obj.id)) + return result diff --git a/docassemble_base/docassemble/base/config.py b/docassemble_base/docassemble/base/config.py index dc09df996..23cd8ac29 100644 --- a/docassemble_base/docassemble/base/config.py +++ b/docassemble_base/docassemble/base/config.py @@ -1,5 +1,7 @@ +# mypy: disable-error-code="attr-defined" import os import re +import copy import sys import socket import threading @@ -12,8 +14,10 @@ import httplib2 from docassemble.base.generate_key import random_string +re._MAXCACHE = 10000 # pylint: disable=protected-access +re._MAXCACHE2 = 5000 # pylint: disable=protected-access START_TIME = time.time() -dbtableprefix = None +dbtableprefix = None # pylint: disable=invalid-name daconfig = {} s3_config = {} S3_ENABLED = False @@ -21,10 +25,10 @@ GC_ENABLED = False azure_config = {} AZURE_ENABLED = False -hostname = None -loaded = False -in_celery = False -in_cron = False +hostname = None # pylint: disable=invalid-name +loaded = False # pylint: disable=invalid-name +in_celery = False # pylint: disable=invalid-name +in_cron = False # pylint: disable=invalid-name errors = [] env_messages = [] allowed = {} @@ -246,12 +250,8 @@ def fix_authorized_domain(domain): def load(**kwargs): - global daconfig - global s3_config global S3_ENABLED - global gc_config global GC_ENABLED - global azure_config global AZURE_ENABLED global DEBUG_BOOT global dbtableprefix @@ -274,7 +274,7 @@ def load(**kwargs): filename = kwargs.get('filename', os.getenv('DA_CONFIG_FILE', '/usr/share/docassemble/config/config.yml')) if 'in_celery' in kwargs and kwargs['in_celery']: in_celery = True - if 'in_cron' in kwargs and kwargs['in_cron']: + if ('in_cron' in kwargs and kwargs['in_cron']) or env_translate('IN_CRON'): in_cron = True if not os.path.isfile(filename): if not os.access(os.path.dirname(filename), os.W_OK): @@ -480,8 +480,11 @@ def load(**kwargs): override_config(daconfig, null_messages, key, env_var, pre_key=['azure']) if env_exists('KUBERNETES'): override_config(daconfig, null_messages, 'kubernetes', 'KUBERNETES') - s3_config = daconfig.get('s3', None) - if not s3_config or ('enable' in s3_config and not s3_config['enable']): + try: + s3_config.update(daconfig.get('s3', {})) + except: + config_error("s3 was not a dict") + if len(s3_config) == 0 or ('enable' in s3_config and not s3_config['enable']): S3_ENABLED = False else: S3_ENABLED = True @@ -489,15 +492,21 @@ def load(**kwargs): s3_config['access key id'] = os.environ['AWSACCESSKEY'] if not s3_config.get('secret access key', None) and env_exists('AWSSECRETACCESSKEY'): s3_config['secret access key'] = os.environ['AWSSECRETACCESSKEY'] - gc_config = daconfig.get('google cloud', None) - if not gc_config or ('enable' in gc_config and not gc_config['enable']) or not ('access key id' in gc_config and gc_config['access key id']) or not ('secret access key' in gc_config and gc_config['secret access key']): + try: + gc_config.update(daconfig.get('google cloud', {})) + except: + config_error("google cloud was not a dict") + if len(gc_config) == 0 or ('enable' in gc_config and not gc_config['enable']) or not ('access key id' in gc_config and gc_config['access key id']) or not ('secret access key' in gc_config and gc_config['secret access key']): GC_ENABLED = False else: GC_ENABLED = True if 'azure' in daconfig and not isinstance(daconfig['azure'], dict): config_error('azure must be a dict') - azure_config = daconfig.get('azure', None) - if not isinstance(azure_config, dict) or ('enable' in azure_config and not azure_config['enable']) or 'account name' not in azure_config or azure_config['account name'] is None or 'account key' not in azure_config or azure_config['account key'] is None: + try: + azure_config.update(daconfig.get('azure', {})) + except: + config_error("azure was not a dict") + if len(azure_config) == 0 or ('enable' in azure_config and not azure_config['enable']) or 'account name' not in azure_config or azure_config['account name'] is None or 'account key' not in azure_config or azure_config['account key'] is None: AZURE_ENABLED = False else: AZURE_ENABLED = True @@ -515,12 +524,14 @@ def load(**kwargs): hostname = os.getenv('SERVERHOSTNAME', socket.gethostname()) if S3_ENABLED: import docassemble.base.amazon # pylint: disable=import-outside-toplevel - cloud = docassemble.base.amazon.s3object(s3_config) + cloud = docassemble.base.amazon.S3Object(s3_config) elif AZURE_ENABLED: import docassemble.base.microsoft # pylint: disable=import-outside-toplevel - cloud = docassemble.base.microsoft.azureobject(azure_config) + cloud = docassemble.base.microsoft.AzureObject(azure_config) if ('key vault name' in azure_config and azure_config['key vault name'] is not None and 'managed identity' in azure_config and azure_config['managed identity'] is not None): - daconfig = cloud.load_with_secrets(daconfig) + daconfig_copy = copy.copy(daconfig) + daconfig.clear() + daconfig.update(cloud.load_with_secrets(daconfig_copy)) else: cloud = None if 'debug startup process' in daconfig and daconfig['debug startup process']: @@ -975,6 +986,12 @@ def load(**kwargs): if new_filename: new_delete_days[new_filename] = days daconfig['interview delete days by filename'] = new_delete_days + if 'celery result retention days' in daconfig: + if not isinstance(daconfig['celery result retention days'], (int, float)) and daconfig['celery result retention days'] >= 0: + config_error('celery result retention days must be an int or a float greater than or equal to zero') + daconfig['celery result retention days'] = 1 + else: + daconfig['celery result retention days'] = 1 for key in ('default interview', 'session list interview', 'dispatch interview', 'auto resume interview'): if key in daconfig: if isinstance(daconfig[key], str): diff --git a/docassemble_base/docassemble/base/core.py b/docassemble_base/docassemble/base/core.py index 8d71f24cc..5c07630d5 100644 --- a/docassemble_base/docassemble/base/core.py +++ b/docassemble_base/docassemble/base/core.py @@ -1,6 +1,36 @@ +# ruff: noqa: F401 +# pylint: disable=unused-import # This module imports names for backwards compatibility and to ensure # that pickled objects in existing sessions can be unpickled. __all__ = ['DAObject', 'DAList', 'DADict', 'DAOrderedDict', 'DASet', 'DAFile', 'DAFileCollection', 'DAFileList', 'DAStaticFile', 'DAEmail', 'DAEmailRecipient', 'DAEmailRecipientList', 'DATemplate', 'DAEmpty', 'DALink', 'RelationshipTree', 'DAContext'] -from docassemble.base.util import DAObject, DAList, DADict, DAOrderedDict, DASet, DAFile, DAFileCollection, DAFileList, DAStaticFile, DAEmail, DAEmailRecipient, DAEmailRecipientList, DATemplate, DAEmpty, DALink, RelationshipTree, DAContext, DAObjectPlusParameters, DACatchAll, RelationshipDir, RelationshipPeer, DALazyTemplate, DALazyTableTemplate, selections, DASessionLocal, DADeviceLocal, DAUserLocal # noqa: F401 # pylint: disable=unused-import +from docassemble.base.util import ( + DAObject, + DAList, + DADict, + DAOrderedDict, + DASet, + DAFile, + DAFileCollection, + DAFileList, + DAStaticFile, + DAEmail, + DAEmailRecipient, + DAEmailRecipientList, + DATemplate, + DAEmpty, + DALink, + RelationshipTree, + DAContext, + DAObjectPlusParameters, + DACatchAll, + RelationshipDir, + RelationshipPeer, + DALazyTemplate, + DALazyTableTemplate, + selections, + DASessionLocal, + DADeviceLocal, + DAUserLocal, +) diff --git a/docassemble_base/docassemble/base/data/questions/examples/def-test.yml b/docassemble_base/docassemble/base/data/questions/examples/def-test.yml new file mode 100644 index 000000000..c0bf1ad18 --- /dev/null +++ b/docassemble_base/docassemble/base/data/questions/examples/def-test.yml @@ -0,0 +1,166 @@ +metadata: + title: Test of defined + example start: 1 + example end: 4 +--- +objects: + list_of_things: DAList.using(object_type=Thing) + list_of_things[i].sub_list: DAList + dict_of_things: DADict.using(object_type=Thing) + dict_of_things[i].sub_list: DAList + list_of_strings: DAList + dict_of_strings: DADict +--- +mandatory: True +code: | + list_of_things + dict_of_things + list_of_strings + dict_of_strings +--- +mandatory: True +code: | + list_of_things.append_object() + list_of_things.append_object() + list_of_things.gathered = True +--- +mandatory: True +code: | + list_of_things[0].name.text = 'First Thing' + list_of_things[0].sub_list.append('First Thing') + list_of_things[0].sub_list.gathered = True +--- +mandatory: True +code: | + list_of_things[1].name.text = 'Second Thing' + list_of_things[1].sub_list.append('First Thing') + list_of_things[1].sub_list.gathered = True +--- +mandatory: True +code: | + dict_of_things.initialize_object('first') + dict_of_things['first'].name.text = 'First Thing' +--- +mandatory: True +code: | + dict_of_things['first'].sub_list.append('First Thing') + dict_of_things['first'].sub_list.gathered = True +--- +mandatory: True +code: | + dict_of_things.initialize_object('second') + dict_of_things['second'].name.text = 'First Thing' + dict_of_things.gathered = True +--- +mandatory: True +code: | + dict_of_things['second'].sub_list.append('First Thing') + dict_of_things['second'].sub_list.gathered = True +--- +mandatory: True +code: | + list_of_strings.append('First Thing') + list_of_strings.append('Second Thing') + list_of_strings.gathered = True +--- +mandatory: True +code: | + dict_of_strings['first'] = 'First Thing' + dict_of_strings['second'] = 'Second Thing' + dict_of_strings.gathered = True +--- +mandatory: True +question: | + Start +continue button field: first_screen +--- +mandatory: True +code: | + assert defined('non_existent') is False + assert value('list_of_things[0].name.text') == 'First Thing' + assert defined('list_of_things[1].name.text') is True + assert defined('list_of_things[2].name.text') is False +--- +mandatory: True +question: | + Testing +subquestion: | + ${ value("dict_of_strings['first']") } + + ${ value('dict_of_strings["first"]') } + + ${ value("list_of_strings[0]") } + + % if defined("dict_of_things"): + Ok + % else: + Not ok: dict_of_things is not defined + % endif + + % if defined("set_of_things"): + Not ok: set_of_things is defined + % else: + Ok + % endif + + ${ showifdef("list_of_things[2]", 'Ok') } + + % if len(list_of_things) > 2: + Not ok: list_of_things greater than 2 + % else: + Ok + % endif + + % if defined("list_of_things[2].sub_list[0]"): + Not ok: list_of_things[2].sub_list[0] is defined + % else: + Ok + % endif + + % if defined("list_of_things[2].sub_list[1]"): + Not ok: list_of_things[2].sub_list[1] is defined + % else: + Ok + % endif + + % if defined("not_a_thing['second'].sub_list[1]"): + Not ok: not_a_thing['second'].sub_list[1] is defined + % else: + Ok + % endif + + % if defined("dict_of_things['second'].sub_list[0]"): + Ok + % else: + Not ok: dict_of_things['second'].sub_list[0] is not defined + % endif + + % if defined("dict_of_things['second'].sub_list[1]"): + Not ok: dict_of_things['second'].sub_list[1] is defined + % else: + Ok + % endif + + % if defined("not_a_thing['second'].sub_list[1]"): + Not ok: not_a_thing['second'].sub_list[1] is defined + % else: + Ok + % endif + + ${ showifdef("dict_of_things['third']", 'Ok') } + + % if 'third' in dict_of_things: + Not ok: 'third' exists in dict_of_things + % else: + Ok + % endif + + % if defined("dict_of_things['third'].sub_list[1]"): + Not ok: dict_of_things['third'].sub_list[1] is defined + % else: + Ok + % endif + + ${ showifdef("dict_of_things['first'].sub_list[0]", "Not ok: dict_of_things['first'].sub_list[0] is undefined") } + + ${ showifdef("dict_of_things['first'].sub_list[1]", 'Ok') } diff --git a/docassemble_base/docassemble/base/data/sources/base-words.yml b/docassemble_base/docassemble/base/data/sources/base-words.yml index 8d7b7a5de..c3aae0689 100644 --- a/docassemble_base/docassemble/base/data/sources/base-words.yml +++ b/docassemble_base/docassemble/base/data/sources/base-words.yml @@ -55,7 +55,6 @@ "API Keys": Null "Apply": Null "App": Null -", are attached": Null "A reset password email has been sent to '%(email)s'. Open that email and follow the instructions to reset your password.": Null "are we": Null "are you": Null @@ -136,6 +135,8 @@ "Chat message you want to send": Null "Check at least one option, or check “%s”": Null "Check box": Null +"checkbox, checked": Null +"checkbox, unchecked": Null "Check if applicable": Null "Chile": Null "China": Null @@ -381,7 +382,6 @@ "Google Drive": Null "Google Drive synchronization": Null "Google Drive Sync": Null -"Google Drive Test": Null "Greece": Null "Greenland": Null "Grenada": Null @@ -500,7 +500,6 @@ "Iraq": Null "Ireland": Null ", is already in use by someone else": Null -", is attached": Null "Isle of Man": Null "is not a valid date and time.": Null "is not a valid date.": Null @@ -701,6 +700,7 @@ "Other settings": Null "other user": Null "other users": Null +"our": Null "ourselves": Null "Output Format": Null "Overrides": Null @@ -1004,7 +1004,6 @@ "(term definition)": Null "Terms used in this question:": Null "Terms used:": Null -"Test": Null "Thailand": Null "Thank you for registering": Null "Thank you for registering with": Null @@ -1330,10 +1329,8 @@ "Your confirmation token has expired.": Null "Your docassemble server": Null "Your document is attached.": Null -"Your document, ": Null "Your documents are attached.": Null "Your document, %s, is attached.": Null -"Your documents, ": Null "Your documents, %s, are attached.": Null "Your documents will be e-mailed to": Null 'Your email address has not yet been confirmed. Check your email Inbox and Spam folders for the confirmation email or Re-send confirmation email.': Null diff --git a/docassemble_base/docassemble/base/dates.py b/docassemble_base/docassemble/base/dates.py new file mode 100644 index 000000000..7a7fe14ec --- /dev/null +++ b/docassemble_base/docassemble/base/dates.py @@ -0,0 +1,484 @@ +try: + import zoneinfo +except ImportError: + from backports import zoneinfo # type: ignore[no-redef] +import datetime +import dateutil +import dateutil.parser +import babel.dates +from .empty import DAEmpty +from .hooks import get_configuration, get_default_timezone +from .language.capitalization import capitalize +from .language.control import get_language +from .language.core import ensure_definition +from .language.language import noun_plural, comma_and_list +from .language.numbers import nice_number +from .language.words import word +from .thread_context import this_thread + +def interview_default(the_part, default_value, language): + if the_part in this_thread.internal and this_thread.internal[the_part] is not None: + return this_thread.internal[the_part] + for lang in (language, get_language(), '*'): + if lang is not None and this_thread.interview is not None and lang in this_thread.interview.default_title and the_part in this_thread.interview.default_title[lang]: + return this_thread.interview.default_title[lang][the_part] + return default_value + +def today(timezone=None, format=None): # pylint: disable=redefined-builtin + """Return today's date at midnight as a DADateTime object. + + Args: + timezone (str or None): IANA timezone name. If None, the interview's + default timezone is used. + format (str or None): If provided, return the date formatted as a + string using this Babel date-format pattern instead of a + DADateTime. + + Returns: + DADateTime or str: Midnight today in the given timezone, or a + formatted date string if ``format`` is specified. + """ + ensure_definition(timezone, format) + if timezone is None: + timezone = get_default_timezone() + val = datetime.datetime.now(datetime.timezone.utc).astimezone(zoneinfo.ZoneInfo(timezone)) + if format is not None: + return dd(val.replace(hour=0, minute=0, second=0, microsecond=0)).format_date(format) + return dd(val.replace(hour=0, minute=0, second=0, microsecond=0)) + + +def babel_language(language): + if 'babel dates map' not in get_configuration(): + return language + return get_configuration()['babel dates map'].get(language, language) + + +def month_of(the_date, as_word=False, language=None): + """Return the month component of a date. + + Args: + the_date (datetime.date, datetime.datetime, or str): The date to + extract the month from. + as_word (bool): If True, return the full month name (e.g. + ``'January'``); otherwise return the month as an integer. + language (str or None): Language code for localizing the month name. + Defaults to the current interview language. + + Returns: + int or str: Month number (1–12) or localized month name. + """ + ensure_definition(the_date, as_word, language) + if language is None: + language = get_language() + try: + if isinstance(the_date, (datetime.datetime, datetime.date)): + date = the_date + else: + date = dateutil.parser.parse(the_date) + if as_word: + return babel.dates.format_date(date, format='MMMM', locale=babel_language(language)) + return int(date.strftime('%m')) + except: + return word("Bad date") + + +def day_of(the_date, language=None): + """Return the day-of-month component of a date. + + Args: + the_date (datetime.date, datetime.datetime, or str): The date to + extract the day from. + language (str or None): Unused; retained for API consistency. + + Returns: + int: Day of the month (1–31). + """ + ensure_definition(the_date, language) + try: + if isinstance(the_date, (datetime.datetime, datetime.date)): + date = the_date + else: + date = dateutil.parser.parse(the_date) + return int(date.strftime('%d')) + except: + return word("Bad date") + + +def dow_of(the_date, as_word=False, language=None): + """Return the day of the week for a date. + + Args: + the_date (datetime.date, datetime.datetime, or str): The date to + inspect. + as_word (bool): If True, return the full weekday name (e.g. + ``'Monday'``); otherwise return an integer from 1 (Monday) to + 7 (Sunday) per ISO 8601. + language (str or None): Language code for localizing the weekday + name. Defaults to the current interview language. + + Returns: + int or str: Day-of-week number or localized weekday name. + """ + ensure_definition(the_date, as_word, language) + if language is None: + language = get_language() + try: + if isinstance(the_date, (datetime.datetime, datetime.date)): + date = the_date + else: + date = dateutil.parser.parse(the_date) + if as_word: + return babel.dates.format_date(date, format='EEEE', locale=babel_language(language)) + return int(date.strftime('%u')) + except: + return word("Bad date") + + +def year_of(the_date, language=None): + """Return the year component of a date. + + Args: + the_date (datetime.date, datetime.datetime, or str): The date to + extract the year from. + language (str or None): Unused; retained for API consistency. + + Returns: + int: Four-digit year. + """ + ensure_definition(the_date, language) + try: + if isinstance(the_date, (datetime.datetime, datetime.date)): + date = the_date + else: + date = dateutil.parser.parse(the_date) + return int(date.strftime('%Y')) + except: + return word("Bad date") + + +def format_date(the_date, format=None, language=None): # pylint: disable=redefined-builtin + """Return a date formatted as a localized string. + + Args: + the_date (datetime.date, datetime.datetime, or str): Date to format. + format (str or None): Babel date-format pattern (e.g. ``'long'``, + ``'short'``, ``'MM/dd/yyyy'``). Defaults to the interview's + configured date format or ``'long'``. + language (str or None): Language/locale code. Defaults to the current + interview language. + + Returns: + str: Formatted date string, or ``''`` for an empty date. + """ + ensure_definition(the_date, format, language) + if isinstance(the_date, DAEmpty): + return "" + if language is None: + language = get_language() + if format is None: + format = interview_default('date format', 'long', language) + try: + if isinstance(the_date, (datetime.datetime, datetime.date)): + date = the_date + else: + date = dateutil.parser.parse(the_date) + return babel.dates.format_date(date, format=format, locale=babel_language(language)) + except: + return word("Bad date") + + +def format_datetime(the_date, format=None, language=None): # pylint: disable=redefined-builtin + """Return a date and time formatted as a localized string. + + Args: + the_date (datetime.datetime or str): Date/time to format. + format (str or None): Babel datetime-format pattern. Defaults to the + interview's configured datetime format or ``'long'``. + language (str or None): Language/locale code. Defaults to the current + interview language. + + Returns: + str: Formatted datetime string, or ``''`` for an empty date. + """ + ensure_definition(the_date, format, language) + if isinstance(the_date, DAEmpty): + return "" + if language is None: + language = get_language() + if format is None: + format = interview_default('datetime format', 'long', language) + try: + if isinstance(the_date, (datetime.datetime, datetime.date)): + date = the_date + else: + date = dateutil.parser.parse(the_date) + return babel.dates.format_datetime(date, format=format, locale=babel_language(language)) + except: + return word("Bad date") + + +def format_time(the_time, format=None, language=None): # pylint: disable=redefined-builtin + """Return a time formatted as a localized string. + + Args: + the_time (datetime.time, datetime.datetime, or str): Time to format. + format (str or None): Babel time-format pattern. Defaults to the + interview's configured time format or ``'short'``. + language (str or None): Language/locale code. Defaults to the current + interview language. + + Returns: + str: Formatted time string, or ``''`` for an empty time. + """ + ensure_definition(the_time, format, language) + if isinstance(the_time, DAEmpty): + return "" + if language is None: + language = get_language() + if format is None: + format = interview_default('time format', 'short', language) + try: + if isinstance(the_time, (datetime.datetime, datetime.date, datetime.time)): + this_time = the_time + else: + this_time = dateutil.parser.parse(the_time) + return babel.dates.format_time(this_time, format=format, locale=babel_language(language)) + except BaseException as errmess: + return word("Bad date: " + str(errmess)) + + +class DateTimeDelta: + + def __str__(self): + return str(self.describe()) + + def describe(self, **kwargs): + specificity = kwargs.get('specificity', None) + output = [] + diff = dateutil.relativedelta.relativedelta(self.end, self.start) + if diff.years != 0: + output.append((abs(diff.years), noun_plural(word('year'), abs(diff.years), noun_is_singular=True))) + if diff.months != 0 and specificity != 'year': + output.append((abs(diff.months), noun_plural(word('month'), abs(diff.months), noun_is_singular=True))) + if diff.days != 0 and specificity not in ('year', 'month'): + output.append((abs(diff.days), noun_plural(word('day'), abs(diff.days), noun_is_singular=True))) + if len(output) == 0 or specificity in ('hour', 'minute', 'second'): + if diff.hours != 0 and specificity not in ('year', 'month', 'day'): + output.append((abs(diff.hours), noun_plural(word('hour'), abs(diff.hours), noun_is_singular=True))) + if (abs(diff.hours) < 2 or specificity in ('minute', 'second')) and diff.minutes != 0 and specificity not in ('year', 'month', 'day', 'hour'): + output.append((abs(diff.minutes), noun_plural(word('minute'), abs(diff.minutes), noun_is_singular=True))) + if len(output) == 0 or specificity == 'second': + if diff.seconds != 0 and specificity not in ('year', 'month', 'day', 'hour', 'minute'): + output.append((abs(diff.seconds), noun_plural(word('second'), abs(diff.seconds), noun_is_singular=True))) + if len(output) == 0: + if specificity is None: + output.append((0, noun_plural(word('second'), 0, noun_is_singular=True))) + else: + output.append((0, noun_plural(word(specificity), 0, noun_is_singular=True))) + if kwargs.get('nice', True): + return_value = comma_and_list(["%s %s" % (nice_number(y[0]), y[1]) for y in output]) + if kwargs.get('capitalize', False): + return capitalize(return_value) + return return_value + return comma_and_list(["%d %s" % y for y in output]) + + +class DADateTime(datetime.datetime): + """A timezone-aware datetime subclass with docassemble-specific formatting and arithmetic. + + Inherits all ``datetime.datetime`` behavior and adds convenience methods + for formatting, date arithmetic, and accessing ISO calendar properties. + + Attributes: + dow (int): Day of the week (1 = Monday … 7 = Sunday, ISO 8601). + week (int): ISO week number of the year. + nanosecond (int): Always 0; provided for compatibility. + """ + + def format(self, format=None, language=None): # pylint: disable=redefined-builtin + return format_date(self, format=format, language=language) + + def format_date(self, format=None, language=None): # pylint: disable=redefined-builtin + return format_date(self, format=format, language=language) + + def format_datetime(self, format=None, language=None): # pylint: disable=redefined-builtin + return format_datetime(self, format=format, language=language) + + def format_time(self, format=None, language=None): # pylint: disable=redefined-builtin + return format_time(self, format=format, language=language) + + def replace_time(self, the_time): + return self.replace(hour=the_time.hour, minute=the_time.minute, second=the_time.second, microsecond=the_time.microsecond) + + @property + def nanosecond(self): + return 0 + + @property + def dow(self): + return self.isocalendar()[2] + + @property + def week(self): + return self.isocalendar()[1] + + def plus(self, **kwargs): + return dd(dt(self) + date_interval(**kwargs)) + + def minus(self, **kwargs): + return dd(dt(self) - date_interval(**kwargs)) + + def __str__(self): + return str(format_date(self)) + + def __add__(self, other): + if isinstance(other, str): + return str(self) + other + val = dt(self) + other + if isinstance(val, datetime.date): + return dd(val) + return val + + def __radd__(self, other): + if isinstance(other, str): + return other + str(self) + return dd(dt(self) + other) + + def __sub__(self, other): + val = dt(self) - other + if isinstance(val, datetime.date): + return dd(val) + return val + + def __rsub__(self, other): + val = other - dt(self) + if isinstance(val, datetime.date): + return dd(val) + return val + + +def current_datetime(timezone=None): + """Return the current date and time as a DADateTime object. + + Args: + timezone (str or None): IANA timezone name. If None, the interview's + default timezone is used. + + Returns: + DADateTime: Current date and time in the specified timezone. + """ + ensure_definition(timezone) + if timezone is None: + timezone = get_default_timezone() + return dd(datetime.datetime.now(datetime.timezone.utc).astimezone(zoneinfo.ZoneInfo(timezone))) + + +def as_datetime(the_date, timezone=None): + """Convert a date or date string to a timezone-aware DADateTime object. + + Args: + the_date (datetime.date, datetime.datetime, or str): Date or + date/time value to convert. String values are parsed with + ``dateutil``. + timezone (str or None): IANA timezone name to attach. If the value + already carries timezone information it is converted to this + zone; otherwise the timezone is applied as-is. Defaults to + the interview's default timezone. + + Returns: + DADateTime: Timezone-aware datetime. + """ + ensure_definition(the_date, timezone) + if timezone is None: + timezone = get_default_timezone() + if isinstance(the_date, datetime.date) and not isinstance(the_date, datetime.datetime): + the_date = datetime.datetime.combine(the_date, datetime.datetime.min.time()) + if isinstance(the_date, datetime.datetime): + new_datetime = the_date + else: + new_datetime = dateutil.parser.parse(the_date) + if new_datetime.tzinfo: + new_datetime = new_datetime.astimezone(zoneinfo.ZoneInfo(timezone)) + else: + new_datetime = new_datetime.replace(tzinfo=zoneinfo.ZoneInfo(timezone)) + return dd(new_datetime) + + +def dd(obj): + if isinstance(obj, DADateTime): + return obj + return DADateTime(obj.year, month=obj.month, day=obj.day, hour=obj.hour, minute=obj.minute, second=obj.second, microsecond=obj.microsecond, tzinfo=obj.tzinfo) + + +def dt(obj): + return datetime.datetime(obj.year, obj.month, obj.day, obj.hour, obj.minute, obj.second, obj.microsecond, obj.tzinfo) + + +def date_interval(**kwargs): + """Return a relative date/time interval. + + All keyword arguments are forwarded to + ``dateutil.relativedelta.relativedelta``. Common arguments include + ``years``, ``months``, ``weeks``, ``days``, ``hours``, ``minutes``, + and ``seconds``. + + Returns: + dateutil.relativedelta.relativedelta: Interval that can be added to + or subtracted from a ``DADateTime`` or ``datetime`` object. + """ + ensure_definition(**kwargs) + return dateutil.relativedelta.relativedelta(**kwargs) + + +def date_difference(starting=None, ending=None, timezone=None): + """Return the difference between two dates. + + Args: + starting (datetime.date, datetime.datetime, str, or None): Start of + the interval. Defaults to the current datetime. + ending (datetime.date, datetime.datetime, str, or None): End of the + interval. Defaults to the current datetime. + timezone (str or None): IANA timezone name used when localizing + naive datetimes. Defaults to the interview's default timezone. + + Returns: + DateTimeDelta: Object with ``weeks``, ``days``, ``hours``, + ``minutes``, ``seconds``, ``years``, and ``delta`` attributes + expressing the difference, and ``start``/``end`` attributes + holding the resolved datetime objects. + """ + ensure_definition(starting, ending, timezone) + if starting is None: + starting = current_datetime() + if ending is None: + ending = current_datetime() + if timezone is None: + timezone = get_default_timezone() + if isinstance(starting, datetime.date) and not isinstance(starting, datetime.datetime): + starting = datetime.datetime.combine(starting, datetime.datetime.min.time()) + if isinstance(ending, datetime.date) and not isinstance(ending, datetime.datetime): + ending = datetime.datetime.combine(ending, datetime.datetime.min.time()) + if not isinstance(starting, datetime.datetime): + starting = dateutil.parser.parse(starting) + if not isinstance(ending, datetime.datetime): + ending = dateutil.parser.parse(ending) + if starting.tzinfo: + starting = starting.astimezone(zoneinfo.ZoneInfo(timezone)) + else: + starting = starting.replace(tzinfo=zoneinfo.ZoneInfo(timezone)) + if ending.tzinfo: + ending = ending.astimezone(zoneinfo.ZoneInfo(timezone)) + else: + ending = ending.replace(tzinfo=zoneinfo.ZoneInfo(timezone)) + delta = ending - starting + output = DateTimeDelta() + output.start = starting + output.end = ending + output.weeks = (delta.days / 7.0) + (delta.seconds / 604800.0) + output.days = delta.days + (delta.seconds / 86400.0) + output.hours = (delta.days * 24.0) + (delta.seconds / 3600.0) + output.minutes = (delta.days * 1440.0) + (delta.seconds / 60.0) + output.seconds = (delta.days * 86400) + delta.seconds + output.years = (delta.days + delta.seconds / 86400.0) / 365.2425 + output.delta = delta + return output diff --git a/docassemble_base/docassemble/base/empty.py b/docassemble_base/docassemble/base/empty.py new file mode 100644 index 000000000..2fb2f1f68 --- /dev/null +++ b/docassemble_base/docassemble/base/empty.py @@ -0,0 +1,199 @@ +class DAEmpty: + """An object that silently absorbs any attribute access or operation. + + DAEmpty avoids triggering errors about missing information by returning + another DAEmpty for any attribute access, returning empty values for + string conversion and length, and absorbing arithmetic operations. + + Attributes: + str (str): The string value returned when the object is converted to + text. Defaults to the empty string. + """ + + def __init__(self, *pargs, **kwargs): # pylint: disable=unused-argument + self.str = str(kwargs.get('str', '')) + + def __getattr__(self, thename): + if thename.startswith('__') or thename == 'str': + return object.__getattribute__(self, thename) + return DAEmpty() + + def __str__(self): + try: + return object.__getattribute__(self, 'str') + except: + return '' + + def __dir__(self): + return [] + + def __contains__(self, item): + return False + + def __iter__(self): + the_list = [] + return the_list.__iter__() + + def __len__(self): + return 0 + + def __reversed__(self): + return [] + + def __getitem__(self, index): + return DAEmpty() + + def __setitem__(self, index, val): + pass + + def __delitem__(self, index): + pass + + def __call__(self, *pargs, **kwargs): + return DAEmpty() + + def __repr__(self): + return repr('') + + def __add__(self, other): + return other + + def __sub__(self, other): + return other + + def __mul__(self, other): + return other + + def __floordiv__(self, other): + return other + + def __mod__(self, other): + return other + + def __divmod__(self, other): + return other + + def __pow__(self, other): + return other + + def __lshift__(self, other): + return other + + def __rshift__(self, other): + return other + + def __and__(self, other): + return other + + def __xor__(self, other): + return other + + def __or__(self, other): + return other + + def __div__(self, other): + return other + + def __truediv__(self, other): + return other + + def __radd__(self, other): + return other + + def __rsub__(self, other): + return other + + def __rmul__(self, other): + return other + + def __rdiv__(self, other): + return other + + def __rtruediv__(self, other): + return other + + def __rfloordiv__(self, other): + return other + + def __rmod__(self, other): + return other + + def __rdivmod__(self, other): + return other + + def __rpow__(self, other): + return other + + def __rlshift__(self, other): + return other + + def __rrshift__(self, other): + return other + + def __rand__(self, other): + return other + + def __ror__(self, other): + return other + + def __neg__(self): + return 0 + + def __pos__(self): + return 0 + + def __abs__(self): + return 0 + + def __invert__(self): + return 0 + + def __complex__(self): + return 0 + + def __int__(self): + return int(0) + + def __float__(self): + return float(0) + + def __oct__(self): + return oct(0) + + def __hex__(self): + return hex(0) + + def __index__(self): + return int(0) + + def __le__(self, other): + return True + + def __ge__(self, other): + return self is other or False + + def __gt__(self, other): + return False + + def __lt__(self, other): + return True + + def __eq__(self, other): + return self is other + + def __ne__(self, other): + return self is not other + + def __hash__(self): + return hash(('',)) + + def as_dict(self): + return self.to_json() + + def to_json(self): + output = {'_class': 'docassemble.base.util.DAEmpty'} + try: + output.update({'str': object.__getattribute__(self, 'str')}) + except Exception: + pass + return output diff --git a/docassemble_base/docassemble/base/error.py b/docassemble_base/docassemble/base/error.py index 487086130..97a7b2d4e 100644 --- a/docassemble_base/docassemble/base/error.py +++ b/docassemble_base/docassemble/base/error.py @@ -106,7 +106,7 @@ def __init__(self, *pargs, **kwargs): if len(the_args) == 0: raise DAError("ForcedNameError must have at least one argument") the_context = {} - the_user_dict = kwargs.get('user_dict', {}) + the_user_dict = kwargs.get('user_dict') or {} for var_name in ('x', 'i', 'j', 'k', 'l', 'm', 'n'): if var_name in the_user_dict: the_context[var_name] = the_user_dict[var_name] diff --git a/docassemble_base/docassemble/base/file_docx.py b/docassemble_base/docassemble/base/file_docx.py index bc19316a5..2b0ff0e22 100644 --- a/docassemble_base/docassemble/base/file_docx.py +++ b/docassemble_base/docassemble/base/file_docx.py @@ -1,92 +1,30 @@ -import re import os -import codecs import time import stat import mimetypes import tempfile -import string import shutil import zipfile from collections import deque -from copy import deepcopy from xml.sax.saxutils import escape as html_escape -from docxtpl import InlineImage, RichText -from docx.shared import Mm, Inches, Pt, Cm, Twips import docx.opc.constants -from docx.oxml.section import CT_SectPr -from docx.oxml.table import CT_Tbl import docx from docxcompose.composer import Composer # For fixing up images, etc when including docx files within templates -import docassemble.base.functions -from docassemble.base.functions import server, package_template_filename, get_config, roman -from docassemble.base.error import DAError -import docassemble.base.filter -import docassemble.base.pandoc -from docassemble.base.logger import logmessage -from bs4 import BeautifulSoup, NavigableString, Tag +from bs4 import NavigableString from pikepdf import Pdf - -zerowidth = '\u200B' +from .error import DAError +from .filter.image_docx import image_for_docx +from .filter.utils import sanitize_xml +from .functions import DALocalFile +from .hooks import fg_make_pdf_for_word_path, fg_make_png_for_pdf_path +from .pandoc import convert_file +from .thread_context import this_thread QPDF_PATH = 'qpdf' NoneType = type(None) DEFAULT_PAGE_WIDTH = '6.5in' -list_types = ['1', 'A', 'a', 'I', 'i'] - - -def fix_double_quote(the_string): - return '"' + re.sub('"', '"', the_string) + '"' - - -class CustomInlineImage(InlineImage): - alt_text = None - - def __init__(self, tpl, image_descriptor, width=None, height=None, anchor=None, alt_text=None): - super().__init__(tpl, image_descriptor, width=width, height=height, anchor=anchor) - self.alt_text = alt_text - - def _insert_image(self): - output = super()._insert_image() - if self.alt_text: - return re.sub(']*>\s*(.*)\s*', r'\1', sanitize_xml(str(first_paragraph._p.xml)), flags=re.DOTALL) - return sanitize_xml(str(sd)) - - for key, val in kwargs.items(): - if hasattr(val, 'instanceName'): - the_repr = val.instanceName - elif isinstance(val, (int, float, bool, NoneType)): - the_repr = val - else: - the_repr = '_codecs.decode(_array.array("b", "' + re.sub(r'\n', '', codecs.encode(bytearray(val, encoding='utf-8'), 'base64').decode()) + '".encode()), "base64").decode()' - first_paragraph.insert_paragraph_before(str("{%%p set %s = %s %%}" % (key, the_repr))) - if 'docx_include_count' not in docassemble.base.functions.this_thread.misc: - docassemble.base.functions.this_thread.misc['docx_include_count'] = 0 - docassemble.base.functions.this_thread.misc['docx_include_count'] += 1 - if single_paragraph: - return re.sub(r']*>\s*(.*)\s*', r'\1', str(first_paragraph._p.xml), flags=re.DOTALL) - return sd - - def get_children(descendants, parsed): subelement = False descendants_buff = deque() @@ -238,489 +88,13 @@ def html_linear_parse(soup): return parsed -def Alpha(number): - multiplier = int((number - 1) / 26) - indexno = (number - 1) % 26 - return string.ascii_uppercase[indexno] * (multiplier + 1) - - -def alpha(number): - multiplier = int((number - 1) / 26) - indexno = (number - 1) % 26 - return string.ascii_lowercase[indexno] * (multiplier + 1) - - -def Roman_Numeral(number): - return roman((number - 1) % 4000, case='upper') - - -def roman_numeral(number): - return roman((number - 1) % 4000, case='lower') - - -class SoupParser: - - def __init__(self, tpl): - self.paragraphs = [{'params': {'style': 'p', 'indentation': 0, 'list_number': 1}, 'runs': [RichText('')]}] - self.current_paragraph = self.paragraphs[-1] - self.run = self.current_paragraph['runs'][-1] - self.bold = False - self.center = False - self.list_number = 1 - self.list_type = list_types[-1] - self.italic = False - self.underline = False - self.strike = False - self.indentation = 0 - self.style = 'p' - self.still_new = True - self.size = None - self.charstyle = None - self.color = None - self.tpl = tpl - - def new_paragraph(self, classes, styles): - if self.still_new: - # logmessage("new_paragraph is still new and style is " + self.style + " and indentation is " + str(self.indentation)) - self.current_paragraph['params']['style'] = self.style - self.current_paragraph['params']['indentation'] = self.indentation - self.set_attribs(classes, styles) - self.list_number += 1 - return - # logmessage("new_paragraph where style is " + self.style + " and indentation is " + str(self.indentation)) - self.current_paragraph = {'params': {'style': self.style, 'indentation': self.indentation, 'list_number': self.list_number}, 'runs': [RichText('')]} - self.set_attribs(classes, styles) - self.list_number += 1 - self.paragraphs.append(self.current_paragraph) - self.run = self.current_paragraph['runs'][-1] - self.still_new = True - - def set_attribs(self, classes, styles): - if 'dacenter' in classes: - self.current_paragraph['params']['align'] = 'center' - elif 'daflushright' in classes: - self.current_paragraph['params']['align'] = 'end' - else: - self.current_paragraph['params']['align'] = 'start' - if len(classes): - if 'daspacingtight' in classes: - self.current_paragraph['params']['spacing'] = 240 - self.current_paragraph['params']['after'] = 0 - elif 'daspacingsingle' in classes: - self.current_paragraph['params']['spacing'] = 240 - self.current_paragraph['params']['after'] = 240 - elif 'daspacingdouble' in classes: - self.current_paragraph['params']['spacing'] = 480 - self.current_paragraph['params']['after'] = 0 - elif 'daspacingoneandahalf' in classes: - self.current_paragraph['params']['spacing'] = 260 - self.current_paragraph['params']['after'] = 0 - elif 'daspacingtriple' in classes: - self.current_paragraph['params']['spacing'] = 700 - self.current_paragraph['params']['after'] = 0 - if styles: - m = re.search(r'margin-left:([0-9\.]+)px', styles) - if m: - self.current_paragraph['params']['leftindent'] = 20 * int(m.group(1)) - m = re.search(r'margin-right:([0-9\.]+)px', styles) - if m: - self.current_paragraph['params']['rightindent'] = 20 * int(m.group(1)) - m = re.search(r'text-indent:([0-9\.]+)px', styles) - if m: - self.current_paragraph['params']['firstline'] = 20 * int(m.group(1)) - - def __str__(self): - output = '' - for para in self.paragraphs: - # logmessage("Got a paragraph where style is " + para['params'].get('style', 'undefined') + " and indentation is " + str(para['params'].get('indentation', 'undefined'))) - output += '' - if 'align' not in para['params']: - para['params']['align'] = 'start' - if para['params']['align'] == 'center': - output += '' - elif para['params']['align'] == 'end': - output += '' - if 'spacing' in para['params']: - output += '' - if para['params']['style'] == 'ul' or para['params']['style'].startswith('ol'): - if 'leftindent' in para['params']: - left_indent = para['params']['leftindent'] - else: - left_indent = 36*para['params']['indentation'] - if 'rightindent' in para['params']: - right_indent = para['params']['rightindent'] - else: - right_indent = 0 - output += '' - elif para['params']['style'] == 'blockquote': - if 'spacing' not in para['params']: - output += '' - output += '' - elif 'leftindent' in para['params'] or 'rightindent' in para['params'] or 'firstline' in para['params']: - if 'leftindent' in para['params']: - left_indent = para['params']['leftindent'] - else: - left_indent = 0 - if 'rightindent' in para['params']: - right_indent = para['params']['rightindent'] - else: - right_indent = 0 - if 'firstline' in para['params']: - first_line = para['params']['firstline'] - else: - first_line = 0 - output += '' - output += '' - if para['params']['style'] == 'ul': - output += str(RichText("•\t")) - if para['params']['style'] == 'ol1': - output += str(RichText(str(para['params']['list_number']) + ".\t")) - elif para['params']['style'] == 'olA': - output += str(RichText(Alpha(para['params']['list_number']) + ".\t")) - elif para['params']['style'] == 'ola': - output += str(RichText(alpha(para['params']['list_number']) + ".\t")) - elif para['params']['style'] == 'olI': - output += str(RichText(Roman_Numeral(para['params']['list_number']) + ".\t")) - elif para['params']['style'] == 'oli': - output += str(RichText(roman_numeral(para['params']['list_number']) + ".\t")) - for run in para['runs']: - output += str(run) - output += '' - return output - - def start_link(self, url): - ref = self.tpl.docx._part.relate_to(url, docx.opc.constants.RELATIONSHIP_TYPE.HYPERLINK, is_external=True) - self.current_paragraph['runs'].append('' % (ref, )) - self.new_run() - self.still_new = False - - def end_link(self): - self.current_paragraph['runs'].append('') - self.new_run() - self.still_new = False - - def new_run(self): - self.current_paragraph['runs'].append(RichText('')) - self.run = self.current_paragraph['runs'][-1] - - def traverse(self, elem): - for part in elem.contents: - if isinstance(part, NavigableString): - self.run.add(str(part), italic=self.italic, bold=self.bold, underline=self.underline, strike=self.strike, size=self.size, style=self.charstyle, color=self.color) - self.still_new = False - elif isinstance(part, Tag): - # logmessage("Part name is " + str(part.name)) - if part.name == 'p': - if 'class' in part.attrs: - classes = part.attrs['class'] - else: - classes = [] - if 'style' in part.attrs: - styles = part.attrs['style'] - else: - styles = "" - self.new_paragraph(classes, styles) - if 'dabold' in classes: - self.bold = True - self.traverse(part) - if 'dabold' in classes: - self.bold = False - elif part.name == 'li': - if 'class' in part.attrs: - classes = part.attrs['class'] - else: - classes = [] - if 'style' in part.attrs: - styles = part.attrs['style'] - else: - styles = "" - self.new_paragraph(classes, styles) - self.traverse(part) - elif part.name == 'ul': - # logmessage("Entering a UL") - oldstyle = self.style - self.style = 'ul' - self.indentation += 10 - self.traverse(part) - self.indentation -= 10 - self.style = oldstyle - # logmessage("Leaving a UL") - elif part.name == 'ol': - # logmessage("Entering a OL") - oldstyle = self.style - oldlistnumber = self.list_number - oldlisttype = self.list_type - if part.get('type', None) in list_types: - self.list_type = part['type'] - else: - self.list_type = list_types[(list_types.index(self.list_type) + 1) % 5] - try: - self.list_number = int(part.get('start', 1)) - except: - self.list_number = 1 - self.style = 'ol' + self.list_type - self.indentation += 10 - self.traverse(part) - self.indentation -= 10 - self.list_type = oldlisttype - self.list_number = oldlistnumber - self.style = oldstyle - # logmessage("Leaving a OL") - elif part.name == 'strong': - self.bold = True - self.traverse(part) - self.bold = False - elif part.name == 'em': - self.italic = True - self.traverse(part) - self.italic = False - elif part.name == 'strike': - self.strike = True - self.traverse(part) - self.strike = False - elif part.name == 'u': - self.underline = True - self.traverse(part) - self.underline = False - elif part.name == 'blockquote': - oldstyle = self.style - self.style = 'blockquote' - self.indentation += 20 - self.traverse(part) - self.indentation -= 20 - self.style = oldstyle - elif re.match(r'h[1-6]', part.name): - oldsize = self.size - self.size = 60 - ((int(part.name[1]) - 1) * 10) - if 'class' in part.attrs: - classes = part.attrs['class'] - else: - classes = [] - if 'style' in part.attrs: - styles = part.attrs['style'] - else: - styles = "" - self.new_paragraph(classes, styles) - self.bold = True - self.traverse(part) - self.bold = False - self.size = oldsize - elif part.name == 'a': - self.start_link(part['href']) - if self.tpl.da_hyperlink_style: - self.charstyle = self.tpl.da_hyperlink_style - else: - self.underline = True - self.color = '#0000ff' - self.traverse(part) - if self.tpl.da_hyperlink_style: - self.charstyle = None - else: - self.underline = False - self.color = None - self.end_link() - elif part.name == 'br': - self.run.add("\n", italic=self.italic, bold=self.bold, underline=self.underline, strike=self.strike, size=self.size, style=self.charstyle, color=self.color) - self.still_new = False - else: - logmessage("Encountered a " + part.__class__.__name__) - - -class InlineSoupParser: - - def __init__(self, tpl): - self.runs = [RichText('')] - self.run = self.runs[-1] - self.bold = False - self.italic = False - self.underline = False - self.indentation = 0 - self.style = 'p' - self.strike = False - self.size = None - self.charstyle = None - self.color = None - self.tpl = tpl - self.at_start = True - self.list_number = 1 - self.list_type = list_types[-1] - - def new_paragraph(self): - if self.at_start: - self.at_start = False - else: - self.run.add("\n", italic=self.italic, bold=self.bold, underline=self.underline, strike=self.strike, size=self.size, style=self.charstyle, color=self.color) - if self.indentation: - self.run.add("\t" * self.indentation) - if self.style == 'ul': - self.run.add("•\t") - if self.style == 'ol1': - self.run.add(str(self.list_number) + ".\t") - self.list_number += 1 - elif self.style == 'olA': - self.run.add(Alpha(self.list_number) + ".\t") - self.list_number += 1 - elif self.style == 'ola': - self.run.add(alpha(self.list_number) + ".\t") - self.list_number += 1 - elif self.style == 'olI': - self.run.add(Roman_Numeral(self.list_number) + ".\t") - self.list_number += 1 - elif self.style == 'oli': - self.run.add(roman_numeral(self.list_number) + ".\t") - self.list_number += 1 - # else: - # self.list_number = 1 - - def __str__(self): - output = '' - for run in self.runs: - output += str(run) - return output - - def start_link(self, url): - ref = self.tpl.docx._part.relate_to(url, docx.opc.constants.RELATIONSHIP_TYPE.HYPERLINK, is_external=True) - self.runs.append('' % (ref, )) - self.new_run() - - def end_link(self): - self.runs.append('') - self.new_run() - - def new_run(self): - self.runs.append(RichText('')) - self.run = self.runs[-1] - - def traverse(self, elem): - for part in elem.contents: - if isinstance(part, NavigableString): - self.run.add(str(part), italic=self.italic, bold=self.bold, underline=self.underline, strike=self.strike, size=self.size, style=self.charstyle, color=self.color) - elif isinstance(part, Tag): - if part.name in ('p', 'blockquote'): - self.new_paragraph() - self.traverse(part) - elif part.name == 'li': - self.new_paragraph() - self.traverse(part) - elif part.name == 'ul': - oldstyle = self.style - self.style = 'ul' - self.indentation += 1 - self.traverse(part) - self.indentation -= 1 - self.style = oldstyle - elif part.name == 'ol': - oldstyle = self.style - oldlistnumber = self.list_number - oldlisttype = self.list_type - if part.get('type', None) in list_types: - self.list_type = part['type'] - else: - self.list_type = list_types[(list_types.index(self.list_type) + 1) % 5] - try: - self.list_number = int(part.get('start', 1)) - except: - self.list_number = 1 - self.style = 'ol' + self.list_type - self.indentation += 1 - self.traverse(part) - self.indentation -= 1 - self.list_type = oldlisttype - self.list_number = oldlistnumber - self.style = oldstyle - elif part.name == 'strong': - self.bold = True - self.traverse(part) - self.bold = False - elif part.name == 'em': - self.italic = True - self.traverse(part) - self.italic = False - elif part.name == 'strike': - self.strike = True - self.traverse(part) - self.strike = False - elif part.name == 'u': - self.underline = True - self.traverse(part) - self.underline = False - elif re.match(r'h[1-6]', part.name): - oldsize = self.size - self.size = 60 - ((int(part.name[1]) - 1) * 10) - self.bold = True - self.traverse(part) - self.bold = False - self.size = oldsize - elif part.name == 'a': - self.start_link(part['href']) - if self.tpl.da_hyperlink_style: - self.charstyle = self.tpl.da_hyperlink_style - else: - self.underline = True - self.color = '#0000ff' - self.traverse(part) - if self.tpl.da_hyperlink_style: - self.charstyle = None - else: - self.underline = False - self.color = None - self.end_link() - elif part.name == 'br': - self.run.add("\n", italic=self.italic, bold=self.bold, underline=self.underline, strike=self.strike, size=self.size, style=self.charstyle, color=self.color) - else: - logmessage("Encountered a " + part.__class__.__name__) - - -def inline_markdown_to_docx(text, question, tpl): - old_context = docassemble.base.functions.this_thread.evaluation_context - docassemble.base.functions.this_thread.evaluation_context = None - try: - text = str(text) - except: - docassemble.base.functions.this_thread.evaluation_context = old_context - raise - docassemble.base.functions.this_thread.evaluation_context = old_context - source_code = docassemble.base.filter.markdown_to_html(text, do_terms=False) - source_code = re.sub(r"\n", ' ', source_code) - source_code = re.sub(r">\s+<", '><', source_code) - soup = BeautifulSoup('' + source_code + '', 'html.parser') - parser = InlineSoupParser(tpl) - for elem in soup.find_all(recursive=False): - parser.traverse(elem) - output = str(parser) - return docassemble.base.filter.docx_template_filter(output, question=question, replace_newlines=False) - - -def markdown_to_docx(text, question, tpl): - old_context = docassemble.base.functions.this_thread.evaluation_context - docassemble.base.functions.this_thread.evaluation_context = None - try: - text = str(text) - except: - docassemble.base.functions.this_thread.evaluation_context = old_context - raise - docassemble.base.functions.this_thread.evaluation_context = old_context - if get_config('new markdown to docx', False): - source_code = docassemble.base.filter.markdown_to_html(text, do_terms=False) - source_code = re.sub(r"\n", ' ', source_code) - source_code = re.sub(r">\s+<", '><', source_code) - soup = BeautifulSoup('' + source_code + '', 'html.parser') - parser = SoupParser(tpl) - for elem in soup.find_all(recursive=False): - parser.traverse(elem) - output = str(parser) - # logmessage(output) - return docassemble.base.filter.docx_template_filter(output, question=question) - return inline_markdown_to_docx(text, question, tpl) - - def pdf_pages(file_info, width): output = '' if width is None: width = DEFAULT_PAGE_WIDTH if not os.path.isfile(file_info['path'] + '.pdf'): if file_info['extension'] in ('rtf', 'doc', 'odt') and not os.path.isfile(file_info['path'] + '.pdf'): - server.fg_make_pdf_for_word_path(file_info['path'], file_info['extension']) + fg_make_pdf_for_word_path(file_info['path'], file_info['extension']) if 'pages' not in file_info: try: with Pdf.open(file_info['path'] + '.pdf') as reader: @@ -741,9 +115,9 @@ def pdf_pages(file_info, width): page_file['path'] = file_info['path'] + 'page-' + formatter % page page_file['fullpath'] = page_file['path'] + '.png' if not os.path.isfile(page_file['fullpath']): - server.fg_make_png_for_pdf_path(file_info['path'] + '.pdf', 'page') + fg_make_png_for_pdf_path(file_info['path'] + '.pdf', 'page') if os.path.isfile(page_file['fullpath']): - output += str(image_for_docx(docassemble.base.functions.DALocalFile(page_file['fullpath']), docassemble.base.functions.this_thread.current_question, docassemble.base.functions.this_thread.misc.get('docx_template', None), width=width)) + output += str(image_for_docx(DALocalFile(page_file['fullpath']), this_thread.current_question, this_thread.misc.get('docx_template', None), width=width)) else: output += "[Error including page image]" output += ' ' @@ -762,7 +136,7 @@ def concatenate_files(path_list): ext = 'doc' else: ext = 'odt' - docassemble.base.pandoc.convert_file(path, new_docx_file.name, ext, 'docx') + convert_file(path, new_docx_file.name, ext, 'docx') new_path_list.append(new_docx_file.name) elif mimetype == 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': new_path_list.append(path) @@ -778,10 +152,6 @@ def concatenate_files(path_list): return docx_file.name -def sanitize_xml(text): - return re.sub(r'{([{%#])', '{' + zerowidth + r'\1', re.sub(r'([}%#])}', r'\1' + zerowidth + '}', text)) - - def fix_docx(path): seen = set() problem_present = False diff --git a/docassemble_base/docassemble/base/filter.py b/docassemble_base/docassemble/base/filter.py deleted file mode 100644 index b6b4ae893..000000000 --- a/docassemble_base/docassemble/base/filter.py +++ /dev/null @@ -1,2006 +0,0 @@ -import re -import os -import mimetypes -import codecs -import json -from io import BytesIO -import tempfile -import time -import stat -import xml.etree.ElementTree as ET -import qrcode -import qrcode.image.svg -from pikepdf import Pdf -import PIL -from docassemble.base.logger import logmessage -from docassemble.base.rtfng.object.picture import Image -from docassemble.base.functions import server, word -import docassemble.base.functions -from docassemble.base import pandoc -from bs4 import BeautifulSoup -from pylatex.utils import escape_latex -from cairosvg import svg2png, svg2eps - -QPDF_PATH = 'qpdf' -NoneType = type(None) - -zerowidth = '\u200B' - -DEFAULT_PAGE_WIDTH = '6.5in' - -term_start = re.compile(r'\[\[') -term_match = re.compile(r'\[\[([^\[\]\|]*)(\|[^\[\]]*)?\]\]', re.DOTALL) -noquote_match = re.compile(r'"') -lt_match = re.compile(r'<') -gt_match = re.compile(r'>') -amp_match = re.compile(r'&') -# amp_match = re.compile(r'&(?!#?[0-9A-Za-z]+;)') -emoji_match = re.compile(r':([A-Za-z][A-Za-z0-9\_\-]+):') -extension_match = re.compile(r'\.[a-z]+$') -map_match = re.compile(r'\[MAP ([^\]]+)\]', flags=re.DOTALL) -code_match = re.compile(r'') - - -def set_default_page_width(width): - global DEFAULT_PAGE_WIDTH - DEFAULT_PAGE_WIDTH = str(width) - - -def get_default_page_width(): - return DEFAULT_PAGE_WIDTH - -DEFAULT_IMAGE_WIDTH = '4in' - - -def set_default_image_width(width): - global DEFAULT_IMAGE_WIDTH - DEFAULT_IMAGE_WIDTH = str(width) - - -def get_default_image_width(): - return DEFAULT_IMAGE_WIDTH - -MAX_HEIGHT_POINTS = 10 * 72 - - -def set_max_height_points(points): - global MAX_HEIGHT_POINTS - MAX_HEIGHT_POINTS = points - - -def get_max_height_points(): - return MAX_HEIGHT_POINTS - -MAX_WIDTH_POINTS = 6.5 * 72.0 - - -def set_max_width_points(points): - global MAX_WIDTH_POINTS - MAX_WIDTH_POINTS = points - - -def get_max_width_points(): - return MAX_WIDTH_POINTS - -# def blank_da_send_mail(*args, **kwargs): -# logmessage("da_send_mail: no mail agent configured!") -# return(None) - -# da_send_mail = blank_da_send_mail - -# def set_da_send_mail(func): -# global da_send_mail -# da_send_mail = func -# return - -# def blank_file_finder(*args, **kwargs): -# return({'filename': "invalid"}) - -# file_finder = blank_file_finder - -# def set_file_finder(func): -# global file_finder -# #logmessage("set the file finder to " + str(func)) -# file_finder = func -# return - -# def blank_url_finder(*args, **kwargs): -# return('about:blank') - -# url_finder = blank_url_finder - -# def set_url_finder(func): -# global url_finder -# url_finder = func -# return - -# def blank_url_for(*args, **kwargs): -# return('about:blank') - -# url_for = blank_url_for - -# def set_url_for(func): -# global url_for -# url_for = func -# return - -rtf_spacing = {'tight': r'\\sl0 ', 'single': r'\\sl0 ', 'oneandahalf': r'\\sl360\\slmult1 ', 'double': r'\\sl480\\slmult1 ', 'triple': r'\\sl720\\slmult1 '} - -rtf_after_space = {'tight': 0, 'single': 1, 'oneandahalf': 0, 'double': 0, 'triplespacing': 0, 'triple': 0} - - -def rtf_prefilter(text): - text = re.sub(r'^# ', '[HEADING1] ', text, flags=re.MULTILINE) - text = re.sub(r'^## ', '[HEADING2] ', text, flags=re.MULTILINE) - text = re.sub(r'^### ', '[HEADING3] ', text, flags=re.MULTILINE) - text = re.sub(r'^#### ', '[HEADING4] ', text, flags=re.MULTILINE) - text = re.sub(r'^##### ', '[HEADING5] ', text, flags=re.MULTILINE) - text = re.sub(r'^###### ', '[HEADING6] ', text, flags=re.MULTILINE) - text = re.sub(r'^####### ', '[HEADING7] ', text, flags=re.MULTILINE) - text = re.sub(r'^######## ', '[HEADING8] ', text, flags=re.MULTILINE) - text = re.sub(r'^######### ', '[HEADING9] ', text, flags=re.MULTILINE) - text = re.sub(r'\s*\[VERTICAL_LINE\]\s*', '\n\n[VERTICAL_LINE]\n\n', text) - text = re.sub(r'\s*\[BREAK\]\s*', '\n\n[BREAK]\n\n', text) - text = re.sub(r'\s+\[END_TWOCOL\]', '\n\n[END_TWOCOL]', text) - text = re.sub(r'\s+\[END_CAPTION\]', '\n\n[END_CAPTION]', text) - text = re.sub(r'\[BEGIN_TWOCOL\]\s+', '[BEGIN_TWOCOL]\n\n', text) - text = re.sub(r'\[BEGIN_CAPTION\]\s+', '[BEGIN_CAPTION]\n\n', text) - return text - - -def repeat_along(chars, match): - output = chars * len(match.group(1)) - # logmessage("Output is " + repr(output)) - return output - - -def rtf_filter(text, metadata=None, styles=None, question=None): - if metadata is None: - metadata = {} - if styles is None: - styles = {} - # logmessage(text) - if 'fontsize' in metadata: - text = re.sub(r'{\\pard', r'\\fs' + str(convert_length(metadata['fontsize'], 'hp')) + r' {\\pard', text, count=1) - after_space_multiplier = convert_length(metadata['fontsize'], 'twips') - else: - after_space_multiplier = 240 - if 'IndentationAmount' in metadata: - indentation_amount = str(convert_length(metadata['IndentationAmount'], 'twips')) - else: - indentation_amount = '720' - if 'Indentation' in metadata: - default_indentation = bool(metadata['Indentation']) - else: - default_indentation = True - if 'SingleSpacing' in metadata and metadata['SingleSpacing']: - # logmessage("Gi there!") - default_spacing = 'single' - if 'Indentation' not in metadata: - default_indentation = False - elif 'OneAndAHalfSpacing' in metadata and metadata['OneAndAHalfSpacing']: - default_spacing = 'oneandahalf' - elif 'DoubleSpacing' in metadata and metadata['DoubleSpacing']: - default_spacing = 'double' - elif 'TripleSpacing' in metadata and metadata['TripleSpacing']: - default_spacing = 'triple' - else: - default_spacing = 'double' - after_space = after_space_multiplier * rtf_after_space[default_spacing] - text = re.sub(r'{\\pard \\ql \\f0 \\sa180 \\li0 \\fi0 \[HEADING([0-9]+)\] *', (lambda x: '{\\pard ' + styles.get(x.group(1), '\\ql \\f0 \\sa180 \\li0 \\fi0 ')), text) - text = re.sub(r'{\\pard \\ql \\f0 \\sa180 \\li0 \\fi0 \[(BEGIN_TWOCOL|BREAK|END_TWOCOL|BEGIN_CAPTION|VERTICAL_LINE|END_CAPTION|TIGHTSPACING|SINGLESPACING|DOUBLESPACING|START_INDENTATION|STOP_INDENTATION|PAGEBREAK|SKIPLINE|NOINDENT|FLUSHLEFT|FLUSHRIGHT|CENTER|BOLDCENTER|INDENTBY[^\]]*)\] *', r'[\1]{\\pard \\ql \\f0 \\sa180 \\li0 \\fi0 ', text) - text = re.sub(r'{\\pard \\ql \\f0 \\sa180 \\li0 \\fi0 *\\par}', r'', text) - text = re.sub(r'\[\[([^\]]*)\]\]', r'\1', text) - # with open('/tmp/asdf.rtf', 'w') as deb_file: - # deb_file.write(text) - text = re.sub(r'\\par}\s*\[(END_TWOCOL|END_CAPTION|BREAK|VERTICAL_LINE)\]', r'}[\1]', text, flags=re.DOTALL) - text = re.sub(r'\[BEGIN_TWOCOL\](.+?)\s*\[BREAK\]\s*(.+?)\[END_TWOCOL\]', rtf_two_col, text, flags=re.DOTALL) - text = re.sub(r'\[EMOJI ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_as_rtf(x, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', lambda x: image_as_rtf(x, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_as_rtf(x, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+)\]', lambda x: image_as_rtf(x, question=question), text) - text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', qr_as_rtf, text) - text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+)\]', qr_as_rtf, text) - text = re.sub(r'\[QR ([^\]]+)\]', qr_as_rtf, text) - text = re.sub(r'\[MAP ([^\]]+)\]', '', text) - text = replace_fields(text) - # text = re.sub(r'\[FIELD ([^\]]+)\]', '', text) - text = re.sub(r'\[TARGET ([^\]]+)\]', '', text) - text = re.sub(r'\[YOUTUBE[^ ]* ([^\]]+)\]', '', text) - text = re.sub(r'\[VIMEO[^ ]* ([^\]]+)\]', '', text) - text = re.sub(r'\[BEGIN_CAPTION\](.+?)\s*\[VERTICAL_LINE\]\s*(.+?)\[END_CAPTION\]', rtf_caption_table, text, flags=re.DOTALL) - text = re.sub(r'\[NBSP\]', r'\\~ ', text) - text = re.sub(r'\[REDACTION_SPACE\]', r'\\u9608\\zwbo', text) - text = re.sub(r'\[REDACTION_WORD ([^\]]+)\]', lambda x: repeat_along('\\u9608', x), text) - text = re.sub(r'\[ENDASH\]', r'{\\endash}', text) - text = re.sub(r'\[EMDASH\]', r'{\\emdash}', text) - text = re.sub(r'\[HYPHEN\]', r'-', text) - text = re.sub(r'\[CHECKBOX\]', r'____', text) - text = re.sub(r'\[BLANK\]', r'________________', text) - text = re.sub(r'\[BLANKFILL\]', r'________________', text) - text = re.sub(r'\[PAGEBREAK\] *', r'\\page ', text) - text = re.sub(r'\[PAGENUM\]', r'{\\chpgn}', text) - text = re.sub(r'\[TOTALPAGES\]', r'{\\field{\\*\\fldinst NUMPAGES } {\\fldrslt 1}}', text) - text = re.sub(r'\[SECTIONNUM\]', r'{\\sectnum}', text) - text = re.sub(r' *\[SKIPLINE\] *', r'\\line ', text) - text = re.sub(r' *\[NEWLINE\] *', r'\\line ', text) - text = re.sub(r' *\[NEWPAR\] *', r'\\par ', text) - text = re.sub(r' *\[BR\] *', r'\\line ', text) - text = re.sub(r' *\[TAB\] *', r'\\tab ', text) - text = re.sub(r' *\[END\] *', r'\n', text) - text = re.sub(r'\\sa180\\sa180\\par', r'\\par', text) - text = re.sub(r'\\sa180', r'\\sa0', text) - text = re.sub(r'(\\trowd \\trgaph[0-9]+)', r'\1\\trqc', text) - text = re.sub(r'\\intbl\\row}\s*{\\pard', r'\\intbl\\row}\n\\line\n{\\pard', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\s*\[(SAVE|RESTORE|TIGHTSPACING|SINGLESPACING|DOUBLESPACING|TRIPLESPACING|ONEANDAHALFSPACING|START_INDENTATION|STOP_INDENTATION)\]\s*', r'\n[\1]\n', text) - lines = text.split('\n') - spacing_command = rtf_spacing[default_spacing] - if default_indentation: - indentation_command = r'\\fi' + str(indentation_amount) + " " - else: - indentation_command = r'\\fi0 ' - text = '' - formatting_stack = [] - for line in lines: - if re.search(r'\[SAVE\]', line): - formatting_stack.append({'spacing_command': spacing_command, 'after_space': after_space, 'default_indentation': default_indentation, 'indentation_command': indentation_command}) - elif re.search(r'\[RESTORE\]', line): - if len(formatting_stack) > 0: - prior_values = formatting_stack.pop() - spacing_command = prior_values['spacing_command'] - after_space = prior_values['after_space'] - default_indentation = prior_values['default_indentation'] - indentation_command = prior_values['indentation_command'] - elif re.search(r'\[TIGHTSPACING\]', line): - spacing_command = rtf_spacing['tight'] - default_spacing = 'tight' - after_space = after_space_multiplier * rtf_after_space[default_spacing] - default_indentation = False - elif re.search(r'\[SINGLESPACING\]', line): - spacing_command = rtf_spacing['single'] - default_spacing = 'single' - after_space = after_space_multiplier * rtf_after_space[default_spacing] - default_indentation = False - elif re.search(r'\[ONEANDAHALFSPACING\]', line): - spacing_command = rtf_spacing['oneandahalf'] - default_spacing = 'oneandahalf' - after_space = after_space_multiplier * rtf_after_space[default_spacing] - elif re.search(r'\[DOUBLESPACING\]', line): - spacing_command = rtf_spacing['double'] - default_spacing = 'double' - after_space = after_space_multiplier * rtf_after_space[default_spacing] - elif re.search(r'\[TRIPLESPACING\]', line): - spacing_command = rtf_spacing['triple'] - default_spacing = 'triple' - after_space = after_space_multiplier * rtf_after_space[default_spacing] - elif re.search(r'\[START_INDENTATION\]', line): - indentation_command = r'\\fi' + str(indentation_amount) + " " - elif re.search(r'\[STOP_INDENTATION\]', line): - indentation_command = r'\\fi0 ' - elif line != '': - special_after_space = None - special_spacing = None - if re.search(r'\[BORDER\]', line): - line = re.sub(r' *\[BORDER\] *', r'', line) - border_text = r'\\box \\brdrhair \\brdrw1 \\brdrcf1 \\brsp29 ' - else: - border_text = r'' - line = re.sub(r'{(\\pard\\intbl \\q[lrc] \\f[0-9]+ \\sa[0-9]+ \\li[0-9]+ \\fi[0-9]+.*?)\\par}', r'\1', line) - if re.search(r'\[NOPAR\]', line): - line = re.sub(r'{\\pard \\ql \\f[0-9]+ \\sa[0-9]+ \\li[0-9]+ \\fi-?[0-9]* *(.*?)\\par}', r'\1', line) - line = re.sub(r' *\[NOPAR\] *', r'', line) - n = re.search(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+) *([0-9\.]+ *[A-Za-z]+)\]', line) - m = re.search(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+)\]', line) - if n: - line = re.sub(r'\\fi-?[0-9]+ ', r'\\fi0 ', line) - line = re.sub(r'\\ri-?[0-9]+ ', r'', line) - line = re.sub(r'\\li-?[0-9]+ ', r'\\li' + str(convert_length(n.group(1), 'twips')) + r' \\ri' + str(convert_length(n.group(2), 'twips')) + ' ', line) - line = re.sub(r'\[INDENTBY[^\]]*\]', '', line) - elif m: - line = re.sub(r'\\fi-?[0-9]+ ', r'\\fi0 ', line) - line = re.sub(r'\\li-?[0-9]+ ', r'\\li' + str(convert_length(m.group(1), 'twips')) + ' ', line) - line = re.sub(r' *\[INDENTBY[^\]]*\] *', '', line) - elif re.search(r'\[NOINDENT\]', line): - line = re.sub(r'\\fi-?[0-9]+ ', r'\\fi0 ', line) - line = re.sub(r' *\[NOINDENT\] *', '', line) - elif re.search(r'\[FLUSHLEFT\]', line): - line = re.sub(r'\\fi-?[0-9]+ ', r'\\fi0 ', line) - line = re.sub(r' *\[FLUSHLEFT\] *', '', line) - special_after_space = after_space_multiplier * 1 - special_spacing = rtf_spacing['single'] - elif re.search(r'\[FLUSHRIGHT\]', line): - line = re.sub(r'\\fi-?[0-9]+ ', r'\\fi0 ', line) - line = re.sub(r'\\ql', r'\\qr', line) - line = re.sub(r' *\[FLUSHRIGHT\] *', '', line) - special_after_space = after_space_multiplier * 1 - special_spacing = rtf_spacing['single'] - elif re.search(r'\[CENTER\]', line): - line = re.sub(r'\\fi-?[0-9]+ ', r'\\fi0 ', line) - line = re.sub(r'\\ql', r'\\qc', line) - line = re.sub(r' *\[CENTER\] *', '', line) - elif re.search(r'\[BOLDCENTER\]', line): - line = re.sub(r'\\fi-?[0-9]+ ', r'\\fi0 ', line) - line = re.sub(r'\\ql', r'\\qc \\b', line) - line = re.sub(r' *\[BOLDCENTER\] *', '', line) - elif indentation_command != '' and not re.search(r'\\widctlpar', line): - line = re.sub(r'\\fi-?[0-9]+ ', indentation_command, line) - if not re.search(r'\\s[0-9]', line): - if special_spacing: - spacing_command_to_use = special_spacing - else: - spacing_command_to_use = spacing_command - line = re.sub(r'\\pard ', r'\\pard ' + str(spacing_command_to_use) + str(border_text), line) - line = re.sub(r'\\pard\\intbl ', r'\\pard\\intbl ' + str(spacing_command_to_use) + str(border_text), line) - if not (re.search(r'\\fi0\\(endash|bullet)', line) or re.search(r'\\s[0-9]', line) or re.search(r'\\intbl', line)): - if special_after_space: - after_space_to_use = special_after_space - else: - after_space_to_use = after_space - if after_space_to_use > 0: - line = re.sub(r'\\sa[0-9]+ ', r'\\sa' + str(after_space_to_use) + ' ', line) - else: - line = re.sub(r'\\sa[0-9]+ ', r'\\sa0 ', line) - text += line + '\n' - text = re.sub(r'{\\pard \\sl[0-9]+\\slmult[0-9]+ \\ql \\f[0-9]+ \\sa[0-9]+ \\li[0-9]+ \\fi-?[0-9]*\s*\\par}', r'', text) - text = re.sub(r'\[MANUALSKIP\]', r'{\\pard \\sl0 \\ql \\f0 \\sa0 \\li0 \\fi0 \\par}', text) - return text - - -def docx_filter(text, metadata=None, question=None): - if metadata is None: - metadata = {} - text = text + "\n\n" - text = re.sub(r'\[\[([^\]]*)\]\]', r'\1', text) - text = re.sub(r'\[EMOJI ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_include_docx(x, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', lambda x: image_include_docx(x, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_include_docx(x, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+)\]', lambda x: image_include_docx(x, question=question), text) - text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', qr_include_docx, text) - text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+)\]', qr_include_docx, text) - text = re.sub(r'\[QR ([^\]]+)\]', qr_include_docx, text) - text = re.sub(r'\[MAP ([^\]]+)\]', '', text) - text = replace_fields(text) - # text = re.sub(r'\[FIELD ([^\]]+)\]', '', text) - text = re.sub(r'\[TARGET ([^\]]+)\]', '', text) - text = re.sub(r'\[YOUTUBE[^ ]* ([^\]]+)\]', '', text) - text = re.sub(r'\[VIMEO[^ ]* ([^\]]+)\]', '', text) - text = re.sub(r'\\clearpage *\\clearpage', '', text) - text = re.sub(r'\[START_INDENTATION\]', '', text) - text = re.sub(r'\[STOP_INDENTATION\]', '', text) - text = re.sub(r'\[BEGIN_CAPTION\](.+?)\[VERTICAL_LINE\]\s*(.+?)\[END_CAPTION\]', '', text, flags=re.DOTALL) - text = re.sub(r'\[BEGIN_TWOCOL\](.+?)\[BREAK\]\s*(.+?)\[END_TWOCOL\]', '', text, flags=re.DOTALL) - text = re.sub(r'\[TIGHTSPACING\] *', '', text) - text = re.sub(r'\[SINGLESPACING\] *', '', text) - text = re.sub(r'\[DOUBLESPACING\] *', '', text) - text = re.sub(r'\[ONEANDAHALFSPACING\] *', '', text) - text = re.sub(r'\[TRIPLESPACING\] *', '', text) - text = re.sub(r'\[NBSP\]', ' ', text) - text = re.sub(r'\[REDACTION_SPACE\]', "\u200B", text) - text = re.sub(r'\[REDACTION_WORD ([^\]]+)\]', lambda x: repeat_along('█', x), text) - text = re.sub(r'\[ENDASH\]', '--', text) - text = re.sub(r'\[EMDASH\]', '---', text) - text = re.sub(r'\[HYPHEN\]', '-', text) - text = re.sub(r'\[CHECKBOX\]', '____', text) - text = re.sub(r'\[BLANK\]', r'__________________', text) - text = re.sub(r'\[BLANKFILL\]', r'__________________', text) - text = re.sub(r'\[PAGEBREAK\] *', '', text) - text = re.sub(r'\[PAGENUM\] *', '', text) - text = re.sub(r'\[TOTALPAGES\] *', '', text) - text = re.sub(r'\[SECTIONNUM\] *', '', text) - text = re.sub(r'\[SKIPLINE\] *', '\n\n', text) - text = re.sub(r'\[VERTICALSPACE\] *', '\n\n', text) - text = re.sub(r'\[NEWLINE\] *', '\n\n', text) - text = re.sub(r'\[NEWPAR\] *', '\n\n', text) - text = re.sub(r'\[BR\] *', '\n\n', text) - text = re.sub(r'\[TAB\] *', '', text) - text = re.sub(r' *\[END\] *', r'\n', text) - text = re.sub(r'\[BORDER\] *(.+?)\n *\n', r'\1\n\n', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[NOINDENT\] *(.+?)\n *\n', r'\1\n\n', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[FLUSHLEFT\] *(.+?)\n *\n', r'\1\n\n', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[FLUSHRIGHT\] *(.+?)\n *\n', r'\1\n\n', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[CENTER\] *(.+?)\n *\n', r'\1\n\n', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[BOLDCENTER\] *(.+?)\n *\n', r'**\1**\n\n', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+)\] *(.+?)\n *\n', r'\2', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+) *([0-9]+ *[A-Za-z]+)\] *(.+?)\n *\n', r'\3', text, flags=re.MULTILINE | re.DOTALL) - return text - - -def docx_template_filter(text, question=None, replace_newlines=True): - # logmessage('docx_template_filter') - if text == 'True': - return True - if text == 'False': - return False - if text == 'None': - return None - text = re.sub(r'\[\[([^\]]*)\]\]', r'\1', text) - text = re.sub(r'\[EMOJI ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_include_docx_template(x, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', lambda x: image_include_docx_template(x, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_include_docx_template(x, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+)\]', lambda x: image_include_docx_template(x, question=question), text) - text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', qr_include_docx_template, text) - text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+)\]', qr_include_docx_template, text) - text = re.sub(r'\[QR ([^\]]+)\]', qr_include_docx_template, text) - text = re.sub(r'\[MAP ([^\]]+)\]', '', text) - text = replace_fields(text) - # text = re.sub(r'\[FIELD ([^\]]+)\]', '', text) - text = re.sub(r'\[TARGET ([^\]]+)\]', '', text) - text = re.sub(r'\[YOUTUBE[^ ]* ([^\]]+)\]', '', text) - text = re.sub(r'\[VIMEO[^ ]* ([^\]]+)\]', '', text) - text = re.sub(r'\\clearpage *\\clearpage', '', text) - text = re.sub(r'\[START_INDENTATION\]', '', text) - text = re.sub(r'\[STOP_INDENTATION\]', '', text) - text = re.sub(r'\[BEGIN_CAPTION\](.+?)\[VERTICAL_LINE\]\s*(.+?)\[END_CAPTION\]', '', text, flags=re.DOTALL) - text = re.sub(r'\[BEGIN_TWOCOL\](.+?)\[BREAK\]\s*(.+?)\[END_TWOCOL\]', '', text, flags=re.DOTALL) - text = re.sub(r'\[TIGHTSPACING\] *', '', text) - text = re.sub(r'\[SINGLESPACING\] *', '', text) - text = re.sub(r'\[DOUBLESPACING\] *', '', text) - text = re.sub(r'\[ONEANDAHALFSPACING\] *', '', text) - text = re.sub(r'\[TRIPLESPACING\] *', '', text) - text = re.sub(r'\[NBSP\]', ' ', text) - text = re.sub(r'\[REDACTION_SPACE\]', "\u200B", text) - # text = re.sub(r'\[REDACTION_SPACE\]', r'', text) - text = re.sub(r'\[REDACTION_WORD ([^\]]+)\]', lambda x: repeat_along('█', x), text) - # text = re.sub(r'\[REDACTION_WORD ([^\]]+)\]', lambda x: repeat_along('X', x), text) - text = re.sub(r'\[ENDASH\]', '--', text) - text = re.sub(r'\[EMDASH\]', '---', text) - text = re.sub(r'\[HYPHEN\]', '-', text) - text = re.sub(r'\\', '', text) - text = re.sub(r'\[CHECKBOX\]', '____', text) - text = re.sub(r'\[BLANK\]', r'__________________', text) - text = re.sub(r'\[BLANKFILL\]', r'__________________', text) - text = re.sub(r'\[PAGEBREAK\] *', '', text) - text = re.sub(r'\[PAGENUM\] *', '', text) - text = re.sub(r'\[TOTALPAGES\] *', '', text) - text = re.sub(r'\[SECTIONNUM\] *', '', text) - text = re.sub(r'\[SKIPLINE\] *', '', text) - text = re.sub(r'\[VERTICALSPACE\] *', '', text) - text = re.sub(r'\[NEWLINE\] *', '', text) - # text = re.sub(r'\n *\n', '[NEWPAR]', text) - if replace_newlines: - text = re.sub(r'\n', ' ', text) - text = re.sub(r'\[NEWPAR\] *', '', text) - text = re.sub(r'\[TAB\] *', '\t', text) - text = re.sub(r'\[NEWPAR\]', '', text) - text = re.sub(r' *\[END\] *', r'', text) - text = re.sub(r'\[BORDER\] *', r'', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[NOINDENT\] *', r'', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[FLUSHLEFT\] *', r'', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[FLUSHRIGHT\] *', r'', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[CENTER\] *', r'', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[BOLDCENTER\] *', r'', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+)\] *(.+?)\n *\n', r'\2', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+) *([0-9]+ *[A-Za-z]+)\] *(.+?)\n *\n', r'\3', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[BR\]', '', text) - text = re.sub(r'\[SKIPLINE\]', '', text) - text = re.sub(r'{([{%#])', '{' + zerowidth + r'\1', re.sub(r'([}%#])}', r'\1' + zerowidth + '}', text)) - return text - - -def metadata_filter(text, doc_format): - if doc_format == 'pdf': - text = re.sub(r'\*\*([^\*]+?)\*\*', r'\\begingroup\\bfseries \1\\endgroup {}', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\*([^\*]+?)\*', r'\\begingroup\\itshape \1\\endgroup {}', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\_\_([^\_]+?)\_\_', r'\\begingroup\\bfseries \1\\endgroup {}', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\_([^\_]+?)\_*', r'\\begingroup\\itshape \1\\endgroup {}', text, flags=re.MULTILINE | re.DOTALL) - return text - - -def redact_latex(match): - return '\\redactword{' + str(escape_latex(match.group(1))) + '}' - - -def pdf_filter(text, metadata=None, question=None): - if metadata is None: - metadata = {} - text = text + "\n\n" - text = re.sub(r'\[\[([^\]]*)\]\]', r'\1', text) - text = re.sub(r'\[EMOJI ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_include_string(x, emoji=True, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', lambda x: image_include_string(x, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_include_string(x, question=question), text) - text = re.sub(r'\[FILE ([^,\]]+)\]', lambda x: image_include_string(x, question=question), text) - text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', qr_include_string, text) - text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+)\]', qr_include_string, text) - text = re.sub(r'\[QR ([^\]]+)\]', qr_include_string, text) - text = re.sub(r'\[MAP ([^\]]+)\]', '', text) - text = replace_fields(text) - # text = re.sub(r'\[FIELD ([^\]]+)\]', '', text) - text = re.sub(r'\[TARGET ([^\]]+)\]', '', text) - text = re.sub(r'\[YOUTUBE[^ ]* ([^\]]+)\]', '', text) - text = re.sub(r'\[VIMEO[^ ]* ([^\]]+)\]', '', text) - text = re.sub(r'\$\$+', '$', text) - text = re.sub(r'\\clearpage *\\clearpage', r'\\clearpage', text) - text = re.sub(r'\[BORDER\]\s*\[(BEGIN_TWOCOL|BEGIN_CAPTION|TIGHTSPACING|SINGLESPACING|DOUBLESPACING|START_INDENTATION|STOP_INDENTATION|NOINDENT|FLUSHLEFT|FLUSHRIGHT|CENTER|BOLDCENTER|INDENTBY[^\]]*)\]', r'[\1] [BORDER]', text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[START_INDENTATION\]', r'\\setlength{\\parindent}{\\myindentamount}\\setlength{\\RaggedRightParindent}{\\parindent}', text) - text = re.sub(r'\[STOP_INDENTATION\]', r'\\setlength{\\parindent}{0in}\\setlength{\\RaggedRightParindent}{\\parindent}', text) - text = re.sub(r'\[BEGIN_CAPTION\](.+?)\[VERTICAL_LINE\]\s*(.+?)\[END_CAPTION\]', pdf_caption, text, flags=re.DOTALL) - text = re.sub(r'\[BEGIN_TWOCOL\](.+?)\[BREAK\]\s*(.+?)\[END_TWOCOL\]', pdf_two_col, text, flags=re.DOTALL) - text = re.sub(r'\[TIGHTSPACING\]\s*', r'\\singlespacing\\setlength{\\parskip}{0pt}\\setlength{\\parindent}{0pt}\\setlength{\\RaggedRightParindent}{\\parindent}', text) - text = re.sub(r'\[SINGLESPACING\]\s*', r'\\singlespacing\\setlength{\\parskip}{\\myfontsize}\\setlength{\\parindent}{0pt}\\setlength{\\RaggedRightParindent}{\\parindent}', text) - text = re.sub(r'\[DOUBLESPACING\]\s*', r'\\doublespacing\\setlength{\\parindent}{\\myindentamount}\\setlength{\\RaggedRightParindent}{\\parindent}', text) - text = re.sub(r'\[ONEANDAHALFSPACING\]\s*', r'\\onehalfspacing\\setlength{\\parindent}{\\myindentamount}\\setlength{\\RaggedRightParindent}{\\parindent}', text) - text = re.sub(r'\[TRIPLESPACING\]\s*', r'\\setlength{\\parindent}{\\myindentamount}\\setlength{\\RaggedRightParindent}{\\parindent}', text) - text = re.sub(r'\[NBSP\]', r'\\myshow{\\nonbreakingspace}', text) - text = re.sub(r'\[REDACTION_SPACE\]', r'\\redactword{~}\\hspace{0pt}', text) - text = re.sub(r'\[REDACTION_WORD ([^\]]+)\]', redact_latex, text) - text = re.sub(r'\[ENDASH\]', r'\\myshow{\\myendash}', text) - text = re.sub(r'\[EMDASH\]', r'\\myshow{\\myemdash}', text) - text = re.sub(r'\[HYPHEN\]', r'\\myshow{\\myhyphen}', text) - text = re.sub(r'\[CHECKBOX\]', r'{\\rule{0.3in}{0.4pt}}', text) - text = re.sub(r'\[BLANK\]', r'\\leavevmode{\\xrfill[-2pt]{0.4pt}}', text) - text = re.sub(r'\[BLANKFILL\]', r'\\leavevmode{\\xrfill[-2pt]{0.4pt}}', text) - text = re.sub(r'\[PAGEBREAK\]\s*', r'\\clearpage ', text) - text = re.sub(r'\[PAGENUM\]', r'\\myshow{\\thepage\\myxspace}', text) - text = re.sub(r'\[TOTALPAGES\]', r'\\myshow{\\pageref*{LastPage}\\myxspace}', text) - text = re.sub(r'\[SECTIONNUM\]', r'\\myshow{\\thesection\\myxspace}', text) - text = re.sub(r'\[VERTICALSPACE\] *', r'\\rule[-24pt]{0pt}{0pt}', text) - text = re.sub(r'\[NEWLINE\] *', r'\\newline ', text) - text = re.sub(r'\[NEWPAR\] *', r'\\par ', text) - text = re.sub(r'\[BR\] *', r'\\manuallinebreak ', text) - text = re.sub(r'\[TAB\] *', r'\\manualindent ', text) - text = re.sub(r' *\[END\] *', r'\n', text) - text = re.sub(r'\[NOINDENT\] *', r'\\noindent ', text) - text = re.sub(r'\[FLUSHLEFT\] *(.+?)\n *\n', flushleft_pdf, text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[FLUSHRIGHT\] *(.+?)\n *\n', flushright_pdf, text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[CENTER\] *(.+?)\n *\n', center_pdf, text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[BOLDCENTER\] *(.+?)\n *\n', boldcenter_pdf, text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+)\] *(.+?)\n *\n', indentby_left_pdf, text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+) *([0-9]+ *[A-Za-z]+)\] *(.+?)\n *\n', indentby_both_pdf, text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\[BORDER\] *(.+?)\n *\n', border_pdf, text, flags=re.MULTILINE | re.DOTALL) - text = re.sub(r'\s*\[SKIPLINE\]\s*', r'\\par\\myskipline ', text) - return text - - -def html_filter(text, status=None, question=None, embedder=None, default_image_width=None, external=False): - if question is None and status is not None: - question = status.question - text = text + "\n\n" - text = re.sub(r'^[|] (.*)$', r'\1
', text, flags=re.MULTILINE) - text = replace_fields(text, status=status, embedder=embedder) - # if embedder is not None: - # text = re.sub(r'\[FIELD ([^\]]+)\]', lambda x: embedder(status, x.group(1)), text) - # else: - # text = re.sub(r'\[FIELD ([^\]]+)\]', 'ERROR: FIELD cannot be used here', text) - text = re.sub(r'\[TARGET ([^\]]+)\]', target_html, text) - if docassemble.base.functions.this_thread.evaluation_context != 'docx': - text = re.sub(r'\[EMOJI ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_url_string(x, emoji=True, question=question, external=external, status=status), text) - text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', lambda x: image_url_string(x, question=question, external=external, status=status), text) - text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_url_string(x, question=question, external=external, status=status), text) - text = re.sub(r'\[FILE ([^,\]]+)\]', lambda x: image_url_string(x, question=question, default_image_width=default_image_width, external=external, status=status), text) - text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', qr_url_string, text) - text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+)\]', qr_url_string, text) - text = re.sub(r'\[QR ([^,\]]+)\]', qr_url_string, text) - if map_match.search(text): - text = map_match.sub((lambda x: map_string(x.group(1), status)), text) - # width="420" height="315" - text = re.sub(r'\[YOUTUBE ([^\]]+)\]', r'
', text) - text = re.sub(r'\[YOUTUBE4:3 ([^\]]+)\]', r'
', text) - text = re.sub(r'\[YOUTUBE16:9 ([^\]]+)\]', r'
', text) - # width="500" height="281" - text = re.sub(r'\[VIMEO ([^\]]+)\]', r'
', text) - text = re.sub(r'\[VIMEO4:3 ([^\]]+)\]', r'
', text) - text = re.sub(r'\[VIMEO16:9 ([^\]]+)\]', r'
', text) - text = re.sub(r'\[BEGIN_CAPTION\](.+?)\[VERTICAL_LINE\]\s*(.+?)\[END_CAPTION\]', html_caption, text, flags=re.DOTALL) - text = re.sub(r'\[BEGIN_TWOCOL\](.+?)\[BREAK\]\s*(.+?)\[END_TWOCOL\]', html_two_col, text, flags=re.DOTALL) - text = re.sub(r'\[NBSP\]', r' ', text) - text = re.sub(r'\[REDACTION_SPACE\]', '█​', text) - text = re.sub(r'\[REDACTION_WORD ([^\]]+)\]', lambda x: repeat_along('█', x), text) - text = re.sub(r'\[ENDASH\]', r'–', text) - text = re.sub(r'\[EMDASH\]', r'—', text) - text = re.sub(r'\[HYPHEN\]', r'-', text) - text = re.sub(r'\[CHECKBOX\]', r'    ', text) - text = re.sub(r'\[BLANK\]', r'            ', text) - text = re.sub(r'\[BLANKFILL\]', r'                              ', text) - text = re.sub(r'\[PAGEBREAK\] *', r'', text) - text = re.sub(r'\[PAGENUM\] *', r'', text) - text = re.sub(r'\[SECTIONNUM\] *', r'', text) - text = re.sub(r'\[SKIPLINE\] *', r'
', text) - text = re.sub(r'\[NEWLINE\] *', r'
', text) - text = re.sub(r'\[NEWPAR\] *', r'

', text) - text = re.sub(r'\[BR\] *', r'
', text) - text = re.sub(r'\[TAB\] *', '', text) - text = re.sub(r' *\[END\] *', r'\n', text) - lines = re.split(r'\n *\n', text) - text = '' - spacing_class = None - doing_indentation = False - for line in lines: - classes = set() - styles = {} - if re.search(r'\[TIGHTSPACING\]', line): - spacing_class = 'daspacingtight' - if re.search(r'\[SINGLESPACING\]', line): - spacing_class = 'daspacingsingle' - if re.search(r'\[DOUBLESPACING\]', line): - spacing_class = 'daspacingdouble' - if re.search(r'\[ONEANDAHALFSPACING\]', line): - spacing_class = 'daspacingoneandahalf' - if re.search(r'\[TRIPLESPACING\]', line): - spacing_class = 'daspacingtriple' - if re.search(r'\[START_INDENTATION\]', line): - doing_indentation = True - if re.search(r'\[STOP_INDENTATION\]', line): - doing_indentation = False - if spacing_class: - classes.add(spacing_class) - if doing_indentation and not re.search(r'\[NOINDENT\]', line): - styles['text-indent'] = '36px' - if re.search(r'\[BORDER\]', line): - classes.add('daborder') - if re.search(r'\[FLUSHLEFT\]', line): - classes.add('daflushleft') - if re.search(r'\[FLUSHRIGHT\]', line): - classes.add('daflushright') - if re.search(r'\[CENTER\]', line): - classes.add('dacenter') - if re.search(r'\[BOLDCENTER\]', line): - classes.add('dacenter') - classes.add('dabold') - m = re.search(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+)\]', line) - if m: - styles["padding-left"] = str(convert_length(m.group(1), 'px')) + 'px' - m = re.search(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+) *([0-9]+ *[A-Za-z]+)\]', line) - if m: - styles["margin-left"] = str(convert_length(m.group(1), 'px')) + 'px' - styles["margin-right"] = str(convert_length(m.group(2), 'px')) + 'px' - orig_length = len(line) - line = re.sub(r'\[(BORDER|NOINDENT|FLUSHLEFT|FLUSHRIGHT|BOLDCENTER|CENTER|TIGHTSPACING|SINGLESPACING|DOUBLESPACING|ONEANDAHALFSPACING|TRIPLESPACING|START_INDENTATION|STOP_INDENTATION)\] *', r'', line) - line = re.sub(r'\[INDENTBY[^\]]*\] *', r'', line) - if orig_length > 0 and len(line) == 0: - continue - if line.startswith('>'): - line = re.sub(r'^> *', '', line) - text += "> " - if len(classes) > 0 or len(styles) > 0: - text += ' 0: - text += ' style="' + "".join(map(lambda x: str(x[0]) + ":" + x[1] + ';', styles.items())) + '"' - text += '>' - text += line + '\n\n' - text = re.sub(r'\n+$', r'', text) - return text - - -def clean_markdown_to_latex(string): - string = re.sub(r'\s*\[SKIPLINE\]\s*', r'\\par\\myskipline ', string) - string = re.sub(r'^[\n ]+', '', string) - string = re.sub(r'[\n ]+$', '', string) - string = re.sub(r' *\n *$', '\n', string) - string = re.sub(r'\n{2,}', '[NEWLINE]', string) - string = re.sub(r'\[BR\]', '[NEWLINE]', string) - string = re.sub(r'\[(NOINDENT|FLUSHLEFT|FLUSHRIGHT|CENTER|BOLDCENTER|TIGHTSPACING|SINGLESPACING|DOUBLESPACING|START_INDENTATION|STOP_INDENTATION|PAGEBREAK)\]\s*', '', string) - string = re.sub(r'\*\*([^\*]+?)\*\*', r'\\textbf{\1}', string) - string = re.sub(r'\*([^\*]+?)\*', r'\\emph{\1}', string) - string = re.sub(r'(?' - - -def target_html(match): - target = match.group(1) - target = re.sub(r'[^A-Za-z0-9\_]', r'', str(target)) - return '' - - -def pdf_two_col(match, add_line=False): - firstcol = clean_markdown_to_latex(match.group(1)) - secondcol = clean_markdown_to_latex(match.group(2)) - if add_line: - return '\\noindent\\begingroup\\singlespacing\\setlength{\\parskip}{0pt}\\mynoindent\\begin{tabular}{@{}m{0.49\\textwidth}|@{\\hspace{1em}}m{0.49\\textwidth}@{}}{' + firstcol + '} & {' + secondcol + '} \\\\ \\end{tabular}\\endgroup\\myskipline' - return '\\noindent\\begingroup\\singlespacing\\setlength{\\parskip}{0pt}\\mynoindent\\begin{tabular}{@{}m{0.49\\textwidth}@{\\hspace{1em}}m{0.49\\textwidth}@{}}{' + firstcol + '} & {' + secondcol + '} \\\\ \\end{tabular}\\endgroup\\myskipline' - - -def html_caption(match): - firstcol = match.group(1) - secondcol = match.group(2) - firstcol = re.sub(r'^\s+', '', firstcol) - firstcol = re.sub(r'\s+$', '', firstcol) - secondcol = re.sub(r'^\s+', '', secondcol) - secondcol = re.sub(r'\s+$', '', secondcol) - firstcol = markdown_to_html(firstcol) - secondcol = markdown_to_html(secondcol) - return '
' + firstcol + '' + secondcol + '
' - - -def html_two_col(match): - firstcol = markdown_to_html(match.group(1)) - secondcol = markdown_to_html(match.group(2)) - return '
' + firstcol + '' + secondcol + '
' - - -def pdf_caption(match): - return pdf_two_col(match, add_line=False) - - -def add_newlines(string): - string = re.sub(r'\[(BR)\]', r'[NEWLINE]', string) - string = re.sub(r' *\n', r'\n', string) - string = re.sub(r'(? 0 and file_info['width'] > 0: - scale = float(pixels)/float(file_info['width']) - # logmessage("scale is " + str(scale)) - if scale*float(file_info['height']) > float(MAX_HEIGHT_POINTS): - scale = float(MAX_HEIGHT_POINTS)/float(file_info['height']) - # logmessage("scale is " + str(scale)) - if scale*float(file_info['width']) > float(MAX_WIDTH_POINTS): - scale = float(MAX_WIDTH_POINTS)/float(file_info['width']) - # logmessage("scale is " + str(scale)) - # scale *= 100.0 - # logmessage("scale is " + str(scale)) - # scale = int(scale) - # logmessage("scale is " + str(scale)) - wtwips = int(scale*float(file_info['width'])*20.0) - htwips = int(scale*float(file_info['height'])*20.0) - image = Image(file_info['fullpath']) - image.Data = re.sub(r'\\picwgoal([0-9]+)', r'\\picwgoal' + str(wtwips), image.Data) - image.Data = re.sub(r'\\pichgoal([0-9]+)', r'\\pichgoal' + str(htwips), image.Data) - else: - image = Image(file_info['fullpath']) - if insert_page_breaks: - content = '\\page ' - else: - content = '' - # logmessage(content + image.Data) - return content + image.Data - -unit_multipliers = {'twips': 0.0500, 'hp': 0.5, 'in': 72, 'pt': 1, 'px': 1, 'em': 12, 'cm': 28.346472} - - -def convert_length(length, unit): - value = pixels_in(length) - if unit in unit_multipliers: - size = float(value)/float(unit_multipliers[unit]) - return int(size) - logmessage("Unit " + str(unit) + " is not a valid unit") - return 300 - - -def pixels_in(length): - m = re.search(r"([0-9.]+) *([a-z]+)", str(length).lower()) - if m: - value = float(m.group(1)) - unit = m.group(2) - # logmessage("value is " + str(value) + " and unit is " + unit) - if unit in unit_multipliers: - size = float(unit_multipliers[unit]) * value - # logmessage("size is " + str(size)) - return int(size) - logmessage("Could not read " + str(length)) - return 300 - - -def image_url_string(match, emoji=False, question=None, default_image_width=None, external=False, status=None): - file_reference = match.group(1) - try: - width = match.group(2) - assert width != 'None' - except: - if default_image_width is not None: - width = default_image_width - else: - width = "300px" - if width == "full": - width = "300px" - if match.lastindex == 3: - if match.group(3) != 'None': - alt_text = 'alt=' + json.dumps(match.group(3)) + ' ' - else: - alt_text = '' - else: - alt_text = '' - return image_url(file_reference, alt_text, width, emoji=emoji, question=question, external=external, status=status) - - -def image_url(file_reference, alt_text, width, emoji=False, question=None, external=False, status=None): - if question and file_reference in question.interview.images: - if status and question.interview.images[file_reference].attribution is not None: - status.attributions.add(question.interview.images[file_reference].attribution) - file_reference = question.interview.images[file_reference].get_reference() - file_info = server.file_finder(file_reference, question=question) - if 'mimetype' in file_info and file_info['mimetype']: - if re.search(r'^audio', file_info['mimetype']): - urls = get_audio_urls([{'text': "[FILE " + file_reference + "]", 'package': None, 'type': 'audio'}], question=question) - if len(urls) > 0: - return audio_control(urls) - return '' - if re.search(r'^video', file_info['mimetype']): - urls = get_video_urls([{'text': "[FILE " + file_reference + "]", 'package': None, 'type': 'video'}], question=question) - if len(urls) > 0: - return video_control(urls) - return '' - if 'extension' in file_info and file_info['extension'] is not None: - if re.match(r'.*%$', width): - width_string = "width:" + width - stack_width_string = width_string - else: - width_string = "max-width:" + width - stack_width_string = "width:" + width - if emoji: - width_string += ';vertical-align: middle' - alt_text = 'alt="" ' - the_url = server.url_finder(file_reference, _question=question, display_filename=file_info['filename'], _external=external) - if the_url is None: - return '[ERROR: File reference ' + str(file_reference) + ' cannot be displayed]' - if width_string == 'width:100%': - extra_class = ' dawideimage' - else: - extra_class = '' - if file_info.get('extension', '') in ('png', 'jpg', 'gif', 'svg', 'jpe', 'jpeg'): - try: - if file_info.get('extension', '') == 'svg': - attributes = ET.parse(file_info['fullpath']).getroot().attrib - layout_width = attributes['width'] - layout_height = attributes['height'] - else: - with PIL.Image.open(file_info['fullpath']) as im: - layout_width, layout_height = im.size - return '' - except: - return '' - if file_info['extension'] in ('pdf', 'docx', 'rtf', 'doc', 'odt'): - if file_info['extension'] in ('docx', 'rtf', 'doc', 'odt') and not os.path.isfile(file_info['path'] + '.pdf'): - server.fg_make_pdf_for_word_path(file_info['path'], file_info['extension']) - server.fg_make_png_for_pdf_path(file_info['path'] + ".pdf", 'screen', page=1) - if re.match(r'[0-9]+', str(file_reference)): - sf = server.SavedFile(int(file_reference), fix=True) - sf.finalize() - if 'pages' not in file_info: - try: - with Pdf.open(file_info['path'] + '.pdf') as reader: - file_info['pages'] = len(reader.pages) - except: - file_info['pages'] = 1 - the_image_url = server.url_finder(file_reference, size="screen", page=1, _question=question, _external=external) - if the_image_url is None: - return '[ERROR: File reference ' + str(file_reference) + ' cannot be displayed]' - if 'filename' in file_info: - title = ' title="' + file_info['filename'] - if 'pages' in file_info and file_info['pages'] > 1: - title += " (" + str(file_info['pages']) + " " + word('pages') + ")" - title += '"' - else: - if 'pages' in file_info and file_info['pages'] > 1: - title = ' title="' + str(file_info['pages']) + " " + word('pages') + '"' - else: - title = '' - if alt_text == '': - the_alt_text = 'alt=' + json.dumps(word("Thumbnail image of document")) + ' ' - else: - the_alt_text = alt_text - try: - with Pdf.open(file_info['path'] + '.pdf') as reader: - layout_width = reader.pages[0].mediabox[2] - reader.pages[0].mediabox[0] - layout_height = reader.pages[0].mediabox[3] - reader.pages[0].mediabox[1] - if width_string == 'width:100%': - output = '' - else: - if 'pages' in file_info and file_info['pages'] >= 1: - extra_pages = min(2, file_info['pages'] - 1) - else: - extra_pages = 2 - aspect_ratio = 1.0*layout_width/layout_height - stack_width_string += "; height: auto; aspect-ratio: " + str(aspect_ratio) + ";" - output = '
' + (('
') * extra_pages) + '
' - except: - output = '' - return output - return '' + file_info['filename'] + '' - return '[Invalid image reference; reference=' + str(file_reference) + ', width=' + str(width) + ', filename=' + file_info.get('filename', 'unknown') + ']' - - -def qr_url_string(match): - string = match.group(1) - try: - width = match.group(2) - assert width != 'None' - except: - width = "300px" - if width == "full": - width = "300px" - if match.lastindex == 3: - if match.group(3) != 'None': - alt_text = str(match.group(3)) - else: - alt_text = word(f"A QR code that goes to {string}") - else: - alt_text = word(f"A QR code that goes to {string}") - width_string = "width:" + width - im = qrcode.make(string, image_factory=qrcode.image.svg.SvgPathFillImage) - output = BytesIO() - im.save(output) - the_image = output.getvalue().decode() - the_image = re.sub(r"<\?xml version='1.0' encoding='UTF-8'\?>\n", '', the_image) - the_image = re.sub(r'height="[0-9]+mm" ', '', the_image) - the_image = re.sub(r'width="[0-9]+mm" ', '', the_image) - m = re.search(r'(viewBox="[^"]+")', the_image) - if m: - viewbox = m.group(1) - else: - viewbox = "" - return '' + the_image + '' + alt_text + '' - - -def convert_pixels(match): - pixels = match.group(1) - return str(int(pixels)/72.0) + "in" - - -def convert_percent(match): - percentage = match.group(1) - return str(float(percentage)/100.0) + '\\textwidth' - - -def image_include_string(match, emoji=False, question=None): - file_reference = match.group(1) - if question and file_reference in question.interview.images: - file_reference = question.interview.images[file_reference].get_reference() - try: - width = match.group(2) - assert width != 'None' - width = re.sub(r'^(.*)px', convert_pixels, width) - width = re.sub(r'^(.*)%', convert_percent, width) - if width == "full": - width = '\\textwidth' - except: - width = DEFAULT_IMAGE_WIDTH - if match.lastindex == 3: - alt_text = match.group(3) - else: - alt_text = None - file_info = server.file_finder(file_reference, question=question) - if 'path' in file_info and 'extension' in file_info: - convert_svg_to_eps(file_info) - if file_info['extension'] == 'gif': - with tempfile.NamedTemporaryFile(prefix="datemp", mode="wb", suffix=".png", delete=False) as png_file: - try: - with PIL.Image.open(file_info['fullpath']) as im: - im.save(png_file.name) - png_file.close() - file_info['path'] = png_file.name - file_info['fullpath'] = png_file.name - file_info['extension'] = 'png' - file_info['mimetype'] = 'image/png' - except BaseException as err: - logmessage("Could not convert GIF to PNG: " + err.__class__.__name__ + ": " + str(err)) - if 'mimetype' in file_info and file_info['mimetype']: - if re.search(r'^(audio|video)', file_info['mimetype']): - return '[reference to file type that cannot be displayed]' - if 'path' in file_info: - if 'extension' in file_info: - if file_info['extension'] in ['png', 'jpg', 'pdf', 'eps', 'jpe', 'jpeg', 'docx', 'rtf', 'doc', 'odt']: - if file_info['extension'] == 'pdf': - output = '\\includepdf[pages={-}]{' + file_info['path'] + '.pdf}' - elif file_info['extension'] in ('docx', 'rtf', 'doc', 'odt'): - if not os.path.isfile(file_info['path'] + '.pdf'): - server.fg_make_pdf_for_word_path(file_info['path'], file_info['extension']) - output = '\\includepdf[pages={-}]{' + file_info['path'] + '.pdf}' - else: - if alt_text: - alt_text_string = ', alt={' + re.sub(r'[{}]', '', alt_text) + '}' - else: - alt_text_string = '' - if emoji: - output = '\\raisebox{-.6\\dp\\strutbox}{\\mbox{\\includegraphics[width=' + width + alt_text_string + ']{' + file_info['path'] + '}}}' - else: - output = '\\mbox{\\includegraphics[width=' + width + alt_text_string + ']{' + file_info['path'] + '}}' - if width == '\\textwidth': - output = '\\clearpage ' + output + '\\clearpage ' - return output - return '[invalid graphics reference]' - - -def image_include_docx(match, question=None): - file_reference = match.group(1) - if question and file_reference in question.interview.images: - file_reference = question.interview.images[file_reference].get_reference() - try: - width = match.group(2) - assert width != 'None' - width = re.sub(r'^(.*)px', convert_pixels, width) - if width == "full": - width = '100%' - except: - width = DEFAULT_IMAGE_WIDTH - if match.lastindex == 3: - alt_text = match.group(3) - else: - alt_text = None - if not alt_text: - alt_text = '' - file_info = server.file_finder(file_reference, question=question) - if 'mimetype' in file_info and file_info['mimetype']: - if re.search(r'^(audio|video)', file_info['mimetype']): - return '[reference to file type that cannot be displayed]' - if 'path' in file_info: - if 'extension' in file_info: - convert_svg_to_eps(file_info) - if file_info['extension'] in ('docx', 'rtf', 'doc', 'odt'): - if not os.path.isfile(file_info['path'] + '.pdf'): - server.fg_make_pdf_for_word_path(file_info['path'], file_info['extension']) - output = '![' + alt_text + '](' + file_info['path'] + '.pdf){width=' + width + '}' - return output - if file_info['extension'] in ['png', 'jpg', 'gif', 'pdf', 'eps', 'jpe', 'jpeg']: - output = '![' + alt_text + '](' + file_info['fullpath'] + '){width=' + width + '}' - return output - return '[invalid graphics reference]' - - -def qr_include_string(match): - string = match.group(1) - try: - width = match.group(2) - assert width != 'None' - width = re.sub(r'^(.*)px', convert_pixels, width) - if width == "full": - width = '\\textwidth' - except: - width = DEFAULT_IMAGE_WIDTH - if match.lastindex == 3: - alt_text = match.group(3) - else: - alt_text = None - im = qrcode.make(string) - with tempfile.NamedTemporaryFile(prefix="datemp", suffix=".png", delete=False) as the_image: - # docassemble.base.functions.this_thread.temporary_resources.add(the_image.name) - im.save(the_image.name) - if alt_text: - alt_text_string = ', alt={' + re.sub(r'[{}]', '', alt_text) + '}' - else: - alt_text_string = '' - output = '\\mbox{\\includegraphics[width=' + width + alt_text_string + ']{' + the_image.name + '}}' - if width == '\\textwidth': - output = '\\clearpage ' + output + '\\clearpage ' - # logmessage("Output is " + output) - return output - - -def qr_include_docx(match): - string = match.group(1) - try: - width = match.group(2) - assert width != 'None' - width = re.sub(r'^(.*)px', convert_pixels, width) - if width == "full": - width = '100%' - except: - width = DEFAULT_IMAGE_WIDTH - if match.lastindex == 3: - alt_text = match.group(3) - else: - alt_text = None - if not alt_text: - alt_text = '' - im = qrcode.make(string) - with tempfile.NamedTemporaryFile(prefix="datemp", suffix=".png", delete=False) as the_image: - # docassemble.base.functions.this_thread.temporary_resources.add(the_image.name) - im.save(the_image.name) - output = '![' + alt_text + '](' + the_image.name + '){width=' + width + '}' - return output - - -def rtf_caption_table(match): - table_text = """\\trowd \\irow0\\irowband0\\lastrow \\ltrrow\\ts24\\trgaph108\\trleft0\\trbrdrt\\brdrs\\brdrw10 \\trbrdrl\\brdrs\\brdrw10 \\trbrdrb\\brdrs\\brdrw10 \\trbrdrr\\brdrs\\brdrw10 \\trbrdrh\\brdrs\\brdrw10 \\trbrdrv\\brdrs\\brdrw10 -\\trftsWidth1\\trftsWidthB3\\trftsWidthA3\\trautofit1\\trpaddl108\\trpaddr108\\trpaddfl3\\trpaddft3\\trpaddfb3\\trpaddfr3\\trcbpat1\\trcfpat1\\tblrsid1508006\\tbllkhdrrows\\tbllkhdrcols\\tbllknocolband\\tblind0\\tblindtype3 \\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrnone -\\clbrdrb\\brdrnone \\clbrdrr\\clshdng0\\brdrs\\brdrw10 \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx4680\\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrs\\brdrw10 \\clbrdrb\\brdrnone \\clbrdrr\\clshdng0\\brdrnone \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx9468\\pard\\plain \\ltrpar -\\ql \\li0\\ri0\\widctlpar\\intbl\\wrapdefault\\aspalpha\\aspnum\\faauto\\adjustright\\rin0\\lin0\\pararsid1508006\\yts24 \\rtlch\\fcs1 \\af0\\afs22\\alang1025 \\ltrch\\fcs0 \\fs22\\lang1033\\langfe1033\\cgrid\\langnp1033\\langfenp1033 { [SAVE][TIGHTSPACING][STOP_INDENTATION]""" + match.group(1) + """}{\\cell}{""" + match.group(2) + """[RESTORE]}{\\cell}\\pard\\plain \\ltrpar -\\ql \\li0\\ri0\\sa200\\sl276\\slmult1\\widctlpar\\intbl\\wrapdefault\\aspalpha\\aspnum\\faauto\\adjustright\\rin0\\lin0 \\rtlch\\fcs1 \\af0\\afs22\\alang1025 \\ltrch\\fcs0 \\fs24\\lang1033\\langfe1033\\cgrid\\langnp1033\\langfenp1033 {\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0 \\insrsid10753242 -\\trowd \\irow0\\irowband0\\lastrow \\ltrrow\\ts24\\trgaph108\\trleft0\\trbrdrt\\brdrs\\brdrw10 \\trbrdrl\\brdrs\\brdrw10 \\trbrdrb\\brdrs\\brdrw10 \\trbrdrr\\brdrs\\brdrw10 \\trbrdrh\\brdrs\\brdrw10 \\trbrdrv\\brdrs\\brdrw10 -\\trftsWidth1\\trftsWidthB3\\trftsWidthA3\\trautofit1\\trpaddl108\\trpaddr108\\trpaddfl3\\trpaddft3\\trpaddfb3\\trpaddfr3\\trcbpat1\\trcfpat1\\tblrsid1508006\\tbllkhdrrows\\tbllkhdrcols\\tbllknocolband\\tblind0\\tblindtype3 \\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrnone -\\clbrdrb\\brdrnone \\clbrdrr\\clshdng0\\brdrs\\brdrw10 \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx4680\\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrs\\brdrw10 \\clbrdrb\\brdrnone \\clbrdrr\\clshdng0\\brdrnone \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx9468\\row }""" - table_text += """\\pard \\ltrpar -\\qc \\li0\\ri0\\sb0\\sl240\\slmult1\\widctlpar\\wrapdefault\\aspalpha\\aspnum\\faauto\\adjustright\\rin0\\lin0\\itap0\\pararsid10753242""" - table_text = re.sub(r'\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0', r'\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0 \\sl240 \\slmult1', table_text) - return table_text + '[MANUALSKIP]' - - -def rtf_two_col(match): - table_text = """\\trowd \\irow0\\irowband0\\lastrow \\ltrrow\\ts24\\trgaph108\\trleft0\\trbrdrt\\brdrs\\brdrw10 \\trbrdrl\\brdrs\\brdrw10 \\trbrdrb\\brdrs\\brdrw10 \\trbrdrr\\brdrs\\brdrw10 \\trbrdrh\\brdrs\\brdrw10 \\trbrdrv\\brdrs\\brdrw10 -\\trftsWidth1\\trftsWidthB3\\trftsWidthA3\\trautofit1\\trpaddl108\\trpaddr108\\trpaddfl3\\trpaddft3\\trpaddfb3\\trpaddfr3\\trcbpat1\\trcfpat1\\tblrsid1508006\\tbllkhdrrows\\tbllkhdrcols\\tbllknocolband\\tblind0\\tblindtype3 \\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrnone -\\clbrdrb\\brdrnone \\clbrdrr\\brdrnone \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx4680\\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrnone \\clbrdrb\\brdrnone \\clbrdrr\\clshdng0\\brdrnone \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx9468\\pard\\plain \\ltrpar -\\ql \\li0\\ri0\\widctlpar\\intbl\\wrapdefault\\aspalpha\\aspnum\\faauto\\adjustright\\rin0\\lin0\\pararsid1508006\\yts24 \\rtlch\\fcs1 \\af0\\afs22\\alang1025 \\ltrch\\fcs0 \\fs22\\lang1033\\langfe1033\\cgrid\\langnp1033\\langfenp1033 {\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0 \\insrsid2427490 [SAVE][TIGHTSPACING][STOP_INDENTATION]""" + match.group(1) + """}{\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0 \\insrsid10753242\\charrsid2427490 \\cell}{""" + match.group(2) + """[RESTORE]}{\\cell}\\pard\\plain \\ltrpar -\\ql \\li0\\ri0\\sa200\\sl276\\slmult1\\widctlpar\\intbl\\wrapdefault\\aspalpha\\aspnum\\faauto\\adjustright\\rin0\\lin0 \\rtlch\\fcs1 \\af0\\afs22\\alang1025 \\ltrch\\fcs0 \\fs24\\lang1033\\langfe1033\\cgrid\\langnp1033\\langfenp1033 {\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0 \\insrsid10753242 -\\trowd \\irow0\\irowband0\\lastrow \\ltrrow\\ts24\\trgaph108\\trleft0\\trbrdrt\\brdrs\\brdrw10 \\trbrdrl\\brdrs\\brdrw10 \\trbrdrb\\brdrs\\brdrw10 \\trbrdrr\\brdrs\\brdrw10 \\trbrdrh\\brdrs\\brdrw10 \\trbrdrv\\brdrs\\brdrw10 -\\trftsWidth1\\trftsWidthB3\\trftsWidthA3\\trautofit1\\trpaddl108\\trpaddr108\\trpaddfl3\\trpaddft3\\trpaddfb3\\trpaddfr3\\trcbpat1\\trcfpat1\\tblrsid1508006\\tbllkhdrrows\\tbllkhdrcols\\tbllknocolband\\tblind0\\tblindtype3 \\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrnone -\\clbrdrb\\brdrnone \\clbrdrr\\brdrnone \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx4680\\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrnone \\clbrdrb\\brdrnone \\clbrdrr\\clshdng0\\brdrnone \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx9468\\row }""" - table_text += """\\pard \\ltrpar -\\qc \\li0\\ri0\\sb0\\sl240\\slmult1\\widctlpar\\wrapdefault\\aspalpha\\aspnum\\faauto\\adjustright\\rin0\\lin0\\itap0\\pararsid10753242""" - table_text = re.sub(r'\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0', r'\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0 \\sl240 \\slmult1', table_text) - return table_text + '[MANUALSKIP]' - - -def get_icon_html(text): - icons_setting = docassemble.base.functions.get_config('default icons', None) - if icons_setting == 'font awesome': - m = re.search(r'^(fa[a-z])-fa-(.*)', text) - if m: - the_prefix = m.group(1) - text = m.group(2) - else: - the_prefix = docassemble.base.functions.get_config('font awesome prefix', 'fa-solid') - if the_prefix == 'fab': - the_prefix = 'fa-brands' - elif the_prefix == 'far': - the_prefix = 'fa-regular' - elif the_prefix == 'fas': - the_prefix = 'fa-solid' - return '' - if icons_setting == 'material icons': - return '' + str(text) + '' - return None - - -def emoji_html(text, status=None, question=None, images=None): - # logmessage("Got to emoji_html") - if status is not None and question is None: - question = status.question - if images is None: - images = question.interview.images - if text in images: - if status is not None and images[text].attribution is not None: - status.attributions.add(images[text].attribution) - return image_url(images[text].get_reference(), word('icon'), '1em', emoji=True, question=question) - icon_html = get_icon_html(text) - if icon_html: - return icon_html - return ":" + str(text) + ":" - - -def emoji_insert(text, status=None, images=None): - if images is None: - images = status.question.interview.images - if text in images: - if status is not None and images[text].attribution is not None: - status.attributions.add(images[text].attribution) - return "[EMOJI " + images[text].get_reference() + ', 1.2em]' - return ":" + str(text) + ":" - - -def link_rewriter(m, status): - the_path = None - if m.group(1).startswith('#'): - return ' 0: - lang = docassemble.base.functions.get_language() - for term in question.terms: - terms_done.add(term.lower()) - # logmessage("Searching for term " + term + " in " + a) - if lang in question.terms[term]['re']: - a = question.terms[term]['re'][lang].sub(sub_term, a) - else: - a = question.terms[term]['re'][question.language].sub(sub_term, a) - # logmessage("string is now " + str(a)) - if len(question.autoterms) > 0: - lang = docassemble.base.functions.get_language() - for term in question.autoterms: - if term.lower() in terms_done: - continue - terms_done.add(term.lower()) - # logmessage("Searching for term " + term + " in " + a) - if lang in question.autoterms[term]['re']: - a = question.autoterms[term]['re'][lang].sub(r'[[\1]]', a) - else: - a = question.autoterms[term]['re'][question.language].sub(r'[[\1]]', a) - # logmessage("string is now " + str(a)) - if 'interview_terms' in status.extras: - interview_terms = status.extras['interview_terms'] - else: - interview_terms = question.interview.terms - if 'interview_autoterms' in status.extras: - interview_autoterms = status.extras['interview_autoterms'] - else: - interview_autoterms = question.interview.autoterms - else: - interview_terms = question.interview.terms - interview_autoterms = question.interview.autoterms - if len(interview_terms) > 0: - lang = docassemble.base.functions.get_language() - if lang in interview_terms and len(interview_terms[lang]) > 0: - for term in interview_terms[lang]: - if term.lower() in terms_done: - continue - terms_done.add(term.lower()) - # logmessage("Searching for term " + term + " in " + a) - a = interview_terms[lang][term]['re'].sub(sub_term, a) - # logmessage("string is now " + str(a)) - elif question.language in interview_terms and len(interview_terms[question.language]) > 0: - for term in interview_terms[question.language]: - if term.lower() in terms_done: - continue - terms_done.add(term.lower()) - # logmessage("Searching for term " + term + " in " + a) - a = interview_terms[question.language][term]['re'].sub(sub_term, a) - # logmessage("string is now " + str(a)) - if len(interview_autoterms) > 0: - lang = docassemble.base.functions.get_language() - if lang in interview_autoterms and len(interview_autoterms[lang]) > 0: - for term in interview_autoterms[lang]: - if term.lower() in terms_done: - continue - terms_done.add(term.lower()) - # logmessage("Searching for term " + term + " in " + a) - a = interview_autoterms[lang][term]['re'].sub(r'[[\1]]', a) - # logmessage("string is now " + str(a)) - elif question.language in interview_autoterms and len(interview_autoterms[question.language]) > 0: - for term in interview_autoterms[question.language]: - if term.lower() in terms_done: - continue - terms_done.add(term.lower()) - # logmessage("Searching for term " + term + " in " + a) - a = interview_autoterms[question.language][term]['re'].sub(r'[[\1]]', a) - # logmessage("string is now " + str(a)) - a = html_filter(str(a), status=status, question=question, embedder=embedder, default_image_width=default_image_width, external=external) - # logmessage("before: " + a) - if status and status.extras.get('tableCssClass', None): - classes = status.extras['tableCssClass'].split(',') - table_class = json.dumps(classes[0].strip()) - if len(classes) > 1: - thead_class = json.dumps(classes[1].strip()) - else: - thead_class = None - else: - table_class = server.default_table_class - thead_class = server.default_thead_class - a = re.sub(r'<(/?)table', r'<\1TABLE', a) - a = re.sub(r'', r'', a) - if use_pandoc: - converter = pandoc.MyPandoc() - converter.output_format = 'html' - converter.input_content = a - converter.convert(question) - result = converter.output_content - else: - try: - result = docassemble.base.functions.this_thread.markdown.reset().convert(a) - except: - # Try again because sometimes it fails randomly and maybe trying again will work. - result = docassemble.base.functions.this_thread.markdown.reset().convert(a) - result = re.sub(r'', r'
', result) - if thead_class: - result = re.sub(r'', r'', result) - result = re.sub(r'
', r'', result) - result = re.sub(r'<(/?)TABLE', r'<\1table', result) - result = re.sub(r'', r'', result) - result = re.sub(r'<(t[dh]) align="(right|left|center)">', r'<\1 class="text-\2">', result) - result = re.sub(r'
', r'
', result) - result = re.sub(r' 0 and 'terms' in status.extras: - result = term_match.sub((lambda x: add_terms(x.group(1), status.extras['terms'], label=x.group(2), status=status, question=question)), result) - if len(question.autoterms) > 0 and 'autoterms' in status.extras: - result = term_match.sub((lambda x: add_terms(x.group(1), status.extras['autoterms'], label=x.group(2), status=status, question=question)), result) - if 'interview_terms' in status.extras: - interview_terms = status.extras['interview_terms'] - else: - interview_terms = question.interview.terms - if 'interview_autoterms' in status.extras: - interview_autoterms = status.extras['interview_autoterms'] - else: - interview_autoterms = question.interview.autoterms - else: - interview_terms = question.interview.terms - interview_autoterms = question.interview.autoterms - if lang in interview_terms and len(interview_terms[lang]): - result = term_match.sub((lambda x: add_terms(x.group(1), interview_terms[lang], label=x.group(2), status=status, question=question)), result) - elif question.language in interview_terms and len(interview_terms[question.language]): - result = term_match.sub((lambda x: add_terms(x.group(1), interview_terms[question.language], label=x.group(2), status=status, question=question)), result) - if lang in interview_autoterms and len(interview_autoterms[lang]): - result = term_match.sub((lambda x: add_terms(x.group(1), interview_autoterms[lang], label=x.group(2), status=status, question=question)), result) - elif question.language in interview_autoterms and len(interview_autoterms[question.language]): - result = term_match.sub((lambda x: add_terms(x.group(1), interview_autoterms[question.language], label=x.group(2), status=status, question=question)), result) - do_not_scan_for_emojis = bool(re.search(r'\[NO_EMOJIS\]', result)) - if do_not_scan_for_emojis: - result = re.sub(r'\[NO_EMOJIS\]\s*', r'', result) - if status is not None and question.interview.scan_for_emojis and not do_not_scan_for_emojis: - result = emoji_match.sub((lambda x: emoji_html(x.group(1), status=status, question=question)), result) - result = re.sub(r'

', result) - if trim: - if result.startswith('

') and result.endswith('

'): - result = re.sub(r'

\s*

', ' ', result[3:-4]) - elif pclass: - result = re.sub('

', '

', result) - if escape: - if escape is True: - result = noquote_match.sub('"', result) - if escape == 'option': - result = re.sub(r'\n\r', ' ', BeautifulSoup(result, 'html.parser').get_text()).strip() - result = lt_match.sub('<', result) - result = gt_match.sub('>', result) - if escape is True: - result = amp_match.sub('&', result) - # logmessage("after: " + result) - # result = result.replace('\n', ' ') - if result: - if strip_newlines: - result = result.replace('\n', ' ') - if divclass is not None: - result = '

' + result + '
' - # if indent and not code_match.search(result): - # return (" " * indent) + re.sub(r'\n', "\n" + (" " * indent), result).rstrip() + "\n" - return result - - -def my_escape(result): - result = noquote_match.sub('"', result) - result = lt_match.sub('<', result) - result = gt_match.sub('>', result) - result = amp_match.sub('&', result) - return result - - -def noquote(string): - # return json.dumps(string.replace('\n', ' ').rstrip()) - return '"' + string.replace('\n', ' ').replace('"', '"').rstrip() + '"' - - -def add_terms_mako(termname, terms, status=None, question=None): - lower_termname = re.sub(r'\s+', ' ', str(termname).lower(), re.DOTALL) - if lower_termname in terms: - term_as_text = to_text(markdown_to_html(str(termname), trim=False, do_terms=False, status=status, question=question), None, None) - return '
' + str(termname) + '' - # logmessage(lower_termname + " is not in terms dictionary") - return '[[' + termname + ']]' - - -def add_terms(termname, terms, label=None, status=None, question=None): - if label is None: - label = str(termname) - else: - label = re.sub(r'^\|', '', label) - lower_termname = re.sub(r'\s+', ' ', termname.lower(), re.DOTALL) - if lower_termname in terms: - term_as_text = to_text(markdown_to_html(label, trim=False, do_terms=False, status=status, question=question), None, None) - return '' + label + '' - return '[[' + termname + ']]' - - -def audio_control(files, preload="metadata", title_text=None): - for d in files: - if isinstance(d, str): - return d - if title_text is None: - title_text = '' - else: - title_text = " title=" + json.dumps(title_text) - output = '' + "\n" - for d in files: - if isinstance(d, list): - output += ' ' - output += "\n" - output += ' ' + word('Listen') + '\n' - output += "\n" - return output - - -def video_control(files): - for d in files: - if isinstance(d, (str, NoneType)): - return str(d) - output = '\n" - return output - - -def get_audio_urls(the_audio, question=None): - output = [] - the_list = [] - to_try = {} - for audio_item in the_audio: - if audio_item['type'] != 'audio': - continue - found_upload = False - pattern = re.compile(r'^\[FILE ([^,\]]+)') - for file_ref in re.findall(pattern, audio_item['text']): - found_upload = True - m = re.match(r'[0-9]+', file_ref) - if m: - file_info = server.file_finder(file_ref, question=question) - if 'path' in file_info: - if file_info['mimetype'] == 'audio/ogg': - output.append([server.url_finder(file_ref, _question=question), file_info['mimetype']]) - elif os.path.isfile(file_info['path'] + '.ogg'): - output.append([server.url_finder(file_ref, ext='ogg', _question=question), 'audio/ogg']) - if file_info['mimetype'] == 'audio/mpeg': - output.append([server.url_finder(file_ref, _question=question), file_info['mimetype']]) - elif os.path.isfile(file_info['path'] + '.mp3'): - output.append([server.url_finder(file_ref, ext='mp3', _question=question), 'audio/mpeg']) - if file_info['mimetype'] not in ['audio/mpeg', 'audio/ogg']: - output.append([server.url_finder(file_ref, _question=question), file_info['mimetype']]) - else: - the_list.append({'text': file_ref, 'package': audio_item['package']}) - if not found_upload: - the_list.append(audio_item) - for audio_item in the_list: - mimetype, encoding = mimetypes.guess_type(audio_item['text']) # pylint: disable=unused-variable - if re.search(r'^http', audio_item['text']): - output.append([audio_item['text'], mimetype]) - continue - basename = os.path.splitext(audio_item['text'])[0] - ext = os.path.splitext(audio_item['text'])[1] - if mimetype not in to_try: - to_try[mimetype] = [] - to_try[mimetype].append({'basename': basename, 'filename': audio_item['text'], 'ext': ext, 'package': audio_item['package']}) - if 'audio/mpeg' in to_try and 'audio/ogg' not in to_try: - to_try['audio/ogg'] = [] - for attempt in to_try['audio/mpeg']: - if attempt['ext'] == '.MP3': - to_try['audio/ogg'].append({'basename': attempt['basename'], 'filename': attempt['basename'] + '.OGG', 'ext': '.OGG', 'package': attempt['package']}) - else: - to_try['audio/ogg'].append({'basename': attempt['basename'], 'filename': attempt['basename'] + '.ogg', 'ext': '.ogg', 'package': attempt['package']}) - if 'audio/ogg' in to_try and 'audio/mpeg' not in to_try: - to_try['audio/mpeg'] = [] - for attempt in to_try['audio/ogg']: - if attempt['ext'] == '.OGG': - to_try['audio/mpeg'].append({'basename': attempt['basename'], 'filename': attempt['basename'] + '.MP3', 'ext': '.MP3', 'package': attempt['package']}) - else: - to_try['audio/mpeg'].append({'basename': attempt['basename'], 'filename': attempt['basename'] + '.mp3', 'ext': '.mp3', 'package': attempt['package']}) - for mimetype in reversed(sorted(to_try.keys())): - for attempt in to_try[mimetype]: - parts = attempt['filename'].split(':') - if len(parts) < 2: - parts = [attempt['package'], attempt['filename']] - if parts[0] is None: - parts[0] = 'None' - parts[1] = re.sub(r'^data/static/', '', parts[1]) - full_file = parts[0] + ':data/static/' + parts[1] - file_info = server.file_finder(full_file, question=question) - if 'fullpath' in file_info: - url = server.url_finder(full_file, _question=question) - output.append([url, mimetype]) - return [item for item in output if item[0] is not None] - - -def get_video_urls(the_video, question=None): - output = [] - the_list = [] - to_try = {} - for video_item in the_video: - if video_item['type'] != 'video': - continue - found_upload = False - if re.search(r'^\[(YOUTUBE|VIMEO)[0-9\:]* ', video_item['text']): - output.append(html_filter(video_item['text'])) - continue - pattern = re.compile(r'^\[FILE ([^,\]]+)') - for file_ref in re.findall(pattern, video_item['text']): - found_upload = True - m = re.match(r'[0-9]+', file_ref) - if m: - file_info = server.file_finder(file_ref, question=question) - if 'path' in file_info: - if file_info['mimetype'] == 'video/ogg': - output.append([server.url_finder(file_ref, _question=question), file_info['mimetype']]) - elif os.path.isfile(file_info['path'] + '.ogv'): - output.append([server.url_finder(file_ref, ext='ogv', _question=question), 'video/ogg']) - if file_info['mimetype'] == 'video/mp4': - output.append([server.url_finder(file_ref, _question=question), file_info['mimetype']]) - elif os.path.isfile(file_info['path'] + '.mp4'): - output.append([server.url_finder(file_ref, ext='mp4', _question=question), 'video/mp4']) - if file_info['mimetype'] not in ['video/mp4', 'video/ogg']: - output.append([server.url_finder(file_ref, _question=question), file_info['mimetype']]) - else: - the_list.append({'text': file_ref, 'package': video_item['package']}) - if not found_upload: - the_list.append(video_item) - for video_item in the_list: - mimetype, encoding = mimetypes.guess_type(video_item['text']) # pylint: disable=unused-variable - if re.search(r'^http', video_item['text']): - output.append([video_item['text'], mimetype]) - continue - basename = os.path.splitext(video_item['text'])[0] - ext = os.path.splitext(video_item['text'])[1] - if mimetype not in to_try: - to_try[mimetype] = [] - to_try[mimetype].append({'basename': basename, 'filename': video_item['text'], 'ext': ext, 'package': video_item['package']}) - if 'video/mp4' in to_try and 'video/ogg' not in to_try: - to_try['video/ogg'] = [] - for attempt in to_try['video/mp4']: - if attempt['ext'] == '.MP4': - to_try['video/ogg'].append({'basename': attempt['basename'], 'filename': attempt['basename'] + '.OGV', 'ext': '.OGV', 'package': attempt['package']}) - else: - to_try['video/ogg'].append({'basename': attempt['basename'], 'filename': attempt['basename'] + '.ogv', 'ext': '.ogv', 'package': attempt['package']}) - if 'video/ogg' in to_try and 'video/mp4' not in to_try: - to_try['video/mp4'] = [] - for attempt in to_try['video/ogg']: - if attempt['ext'] == '.OGV': - to_try['video/mp4'].append({'basename': attempt['basename'], 'filename': attempt['basename'] + '.MP4', 'ext': '.MP4', 'package': attempt['package']}) - else: - to_try['audio/mpeg'].append({'basename': attempt['basename'], 'filename': attempt['basename'] + '.mp4', 'ext': '.mp4', 'package': attempt['package']}) - for mimetype in reversed(sorted(to_try.keys())): - for attempt in to_try[mimetype]: - parts = attempt['filename'].split(':') - if len(parts) < 2: - parts = [attempt['package'], attempt['filename']] - parts[1] = re.sub(r'^data/static/', '', parts[1]) - if parts[0] is None: - full_file = 'data/static/' + parts[1] - else: - full_file = parts[0] + ':data/static/' + parts[1] - file_info = server.file_finder(full_file, question=question) - if 'fullpath' in file_info: - url = server.url_finder(full_file, _question=question) - if url is not None: - output.append([url, mimetype]) - return output - - -def process_target(text): - return re.sub(r'\[TARGET ([^\]]+)\]', target_html, text) - - -def to_text(html_doc, terms, links): - output = "" - # logmessage("to_text: html doc is " + str(html_doc)) - if not html_doc.startswith('<'): - html_doc = "" + html_doc + "" - soup = BeautifulSoup(html_doc, 'html.parser') - [s.extract() for s in soup(['style', 'script', '[document]', 'head', 'title', 'audio', 'video', 'pre', 'attribution'])] # pylint: disable=expression-not-assigned - [s.extract() for s in soup.find_all(hidden)] # pylint: disable=expression-not-assigned - [s.extract() for s in soup.find_all('div', {'class': 'dainvisible'})] # pylint: disable=expression-not-assigned - for s in soup.find_all(do_show): - if s.name in ['input', 'textarea', 'img'] and s.has_attr('alt'): - words = s.attrs['alt'] - if s.has_attr('placeholder'): - words += ", " + s.attrs['placeholder'] - else: - words = s.get_text() - words = re.sub(r'\n\s*', ' ', words, flags=re.DOTALL) - output += words + "\n" - for s in soup.find_all('a'): - if s.has_attr('class') and s.attrs['class'][0] == 'daterm' and s.has_attr('data-bs-content'): - terms[s.string] = s.attrs['data-bs-content'] - elif s.has_attr('href'): # and (s.attrs['href'].startswith(url) or s.attrs['href'].startswith('?')): - # logmessage("Adding a link: " + s.attrs['href']) - links.append((s.attrs['href'], s.get_text())) - output = re.sub(br'\u201c'.decode('raw_unicode_escape'), '"', output) - output = re.sub(br'\u201d'.decode('raw_unicode_escape'), '"', output) - output = re.sub(br'\u2018'.decode('raw_unicode_escape'), "'", output) - output = re.sub(br'\u2019'.decode('raw_unicode_escape'), "'", output) - output = re.sub(br'\u201b'.decode('raw_unicode_escape'), "'", output) - output = re.sub(r'&gt;', '>', output) - output = re.sub(r'&lt;', '<', output) - output = re.sub(r'>', '>', output) - output = re.sub(r'<', '<', output) - output = re.sub(r'<[^>]+>', '', output) - output = re.sub(r'\n$', '', output) - output = re.sub(r' +', ' ', output) - return output - -bad_list = ['div', 'option'] - -good_list = ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'button', 'textarea', 'note'] - - -def do_show(element): - if re.match('', str(element), re.DOTALL): - return False - if element.name in ['option'] and element.has_attr('selected'): - return True - if element.name in bad_list: - return False - if element.name in ['img', 'input'] and element.has_attr('alt'): - return True - if element.name in good_list: - return True - if element.parent and element.parent.name in good_list: - return False - if element.string: - return True - if re.match(r'\s+', element.get_text()): - return False - return False - - -def hidden(element): - if element.name == 'input': - if element.has_attr('type'): - if element.attrs['type'] == 'hidden': - return True - return False - - -def replace_fields(string, status=None, embedder=None): - if not re.search(r'\[FIELD ', string): - return string - matches = [] - in_match = False - start_match = None - depth = 0 - i = 0 - while i < len(string): - if string[i:i+7] == '[FIELD ': - in_match = True - start_match = i - i += 7 - continue - if in_match: - if string[i] == '[': - depth += 1 - elif string[i] == ']': - if depth == 0: - i += 1 - matches.append((start_match, i)) - in_match = False - continue - depth -= 1 - i += 1 - - field_strings = [] - for (start, end) in matches: - field_strings.append(string[start:end]) - # logmessage(repr(field_strings)) - for field_string in field_strings: - if embedder is None: - string = string.replace(field_string, 'ERROR: FIELD cannot be used here') - else: - string = string.replace(field_string, embedder(status, field_string)) - return string - - -def image_include_docx_template(match, question=None): - file_reference = match.group(1) - if question and file_reference in question.interview.images: - file_reference = question.interview.images[file_reference].get_reference() - try: - width = match.group(2) - assert width != 'None' - width = re.sub(r'^(.*)px', convert_pixels, width) - if width == "full": - width = '100%' - except: - width = DEFAULT_IMAGE_WIDTH - if match.lastindex == 3: - alt_text = match.group(3) - else: - alt_text = None - file_info = server.file_finder(file_reference, question=question) - if 'mimetype' in file_info and file_info['mimetype']: - if re.search(r'^(audio|video)', file_info['mimetype']): - return '[reference to file type that cannot be displayed]' - if 'path' in file_info: - convert_svg_to_eps(file_info) - if 'mimetype' in file_info and file_info['mimetype']: - if file_info['mimetype'] in ('text/markdown', 'text/plain'): - with open(file_info['fullpath'], 'r', encoding='utf-8') as f: - contents = f.read() - if file_info['mimetype'] == 'text/plain': - return contents - return docassemble.base.file_docx.markdown_to_docx(contents, question, docassemble.base.functions.this_thread.misc.get('docx_template', None)) - if file_info['mimetype'] == 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': - return str(docassemble.base.file_docx.include_docx_template(docassemble.base.functions.DALocalFile(file_info['fullpath']))) - return str(docassemble.base.file_docx.image_for_docx(file_reference, question, docassemble.base.functions.this_thread.misc.get('docx_template', None), width=width, alt_text=alt_text)) - return '[reference to file that could not be found]' - - -def qr_include_docx_template(match): - string = match.group(1) - try: - width = match.group(2) - assert width != 'None' - width = re.sub(r'^(.*)px', convert_pixels, width) - if width == "full": - width = '100%' - except: - width = DEFAULT_IMAGE_WIDTH - if match.lastindex == 3: - alt_text = match.group(3) - else: - alt_text = None - im = qrcode.make(string) - with tempfile.NamedTemporaryFile(prefix="datemp", suffix=".png", delete=False) as the_image: - im.save(the_image.name) - return str(docassemble.base.file_docx.image_for_docx(docassemble.base.functions.DALocalFile(the_image.name), None, docassemble.base.functions.this_thread.misc.get('docx_template', None), width=width, alt_text=alt_text)) - - -def convert_svg_to_eps(file_info): - try: - if file_info['extension'] == 'svg': - with tempfile.NamedTemporaryFile(prefix="datemp", mode="wb", suffix=".eps", delete=False) as eps_file: - with open(file_info['fullpath'], 'rb') as fp: - svg2eps(file_obj=fp, write_to=eps_file) - file_info['path'] = eps_file.name - file_info['fullpath'] = eps_file.name - file_info['extension'] = 'eps' - file_info['mimetype'] = 'application/postscript' - eps_file.close() - except BaseException as err: - logmessage("Failure to convert SVG to EPS: " + err.__class__.__name__ + ": " + str(err)) - - -def convert_svg_to_png(file_info): - try: - if file_info['extension'] == 'svg': - with tempfile.NamedTemporaryFile(prefix="datemp", mode="wb", suffix=".png", delete=False) as png_file: - with open(file_info['fullpath'], 'rb') as fp: - svg2png(file_obj=fp, write_to=png_file, dpi=300) - png_file.flush() - with PIL.Image.open(png_file.name) as im: - file_info['width'], file_info['height'] = im.size - file_info['path'] = png_file.name - file_info['fullpath'] = png_file.name - file_info['extension'] = 'png' - file_info['mimetype'] = 'image/png' - png_file.close() - except BaseException as err: - logmessage("Failure to convert SVG to PNG: " + err.__class__.__name__ + ": " + str(err)) diff --git a/docassemble_webapp/docassemble/webapp/templates/pages/__init__.py b/docassemble_base/docassemble/base/filter/__init__.py similarity index 100% rename from docassemble_webapp/docassemble/webapp/templates/pages/__init__.py rename to docassemble_base/docassemble/base/filter/__init__.py diff --git a/docassemble_base/docassemble/base/filter/docx.py b/docassemble_base/docassemble/base/filter/docx.py new file mode 100644 index 000000000..0fef46c66 --- /dev/null +++ b/docassemble_base/docassemble/base/filter/docx.py @@ -0,0 +1,803 @@ +import re +import os +import codecs +from copy import deepcopy +import tempfile +import string +from bs4 import BeautifulSoup, NavigableString, Tag +import docx +import qrcode +import qrcode.image.svg +from docxtpl import RichText +from ..config import daconfig +from ..functions import DALocalFile, roman, package_template_filename +from ..hooks import file_finder, fg_make_pdf_for_word_path +from ..logger import logmessage +from ..thread_context import this_thread +from .docx_subdoc import fix_subdoc +from .html import markdown_to_html +from .image_docx import image_for_docx +from .utils import ( + repeat_along, + replace_fields, + convert_pixels, + get_default_image_width, + convert_svg_to_eps, + list_types, + zerowidth, + sanitize_xml, +) + +NoneType = type(None) + +def docx_filter(text, metadata=None, question=None): + if metadata is None: + metadata = {} + text = text + "\n\n" + text = re.sub(r'\[\[([^\]]*)\]\]', r'\1', text) + text = re.sub(r'\[EMOJI ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_include_docx(x, question=question), text) + text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', lambda x: image_include_docx(x, question=question), text) + text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_include_docx(x, question=question), text) + text = re.sub(r'\[FILE ([^,\]]+)\]', lambda x: image_include_docx(x, question=question), text) + text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', qr_include_docx, text) + text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+)\]', qr_include_docx, text) + text = re.sub(r'\[QR ([^\]]+)\]', qr_include_docx, text) + text = re.sub(r'\[MAP ([^\]]+)\]', '', text) + text = replace_fields(text) + # text = re.sub(r'\[FIELD ([^\]]+)\]', '', text) + text = re.sub(r'\[TARGET ([^\]]+)\]', '', text) + text = re.sub(r'\[YOUTUBE[^ ]* ([^\]]+)\]', '', text) + text = re.sub(r'\[VIMEO[^ ]* ([^\]]+)\]', '', text) + text = re.sub(r'\\clearpage *\\clearpage', '', text) + text = re.sub(r'\[START_INDENTATION\]', '', text) + text = re.sub(r'\[STOP_INDENTATION\]', '', text) + text = re.sub(r'\[BEGIN_CAPTION\](.+?)\[VERTICAL_LINE\]\s*(.+?)\[END_CAPTION\]', '', text, flags=re.DOTALL) + text = re.sub(r'\[BEGIN_TWOCOL\](.+?)\[BREAK\]\s*(.+?)\[END_TWOCOL\]', '', text, flags=re.DOTALL) + text = re.sub(r'\[TIGHTSPACING\] *', '', text) + text = re.sub(r'\[SINGLESPACING\] *', '', text) + text = re.sub(r'\[DOUBLESPACING\] *', '', text) + text = re.sub(r'\[ONEANDAHALFSPACING\] *', '', text) + text = re.sub(r'\[TRIPLESPACING\] *', '', text) + text = re.sub(r'\[NBSP\]', ' ', text) + text = re.sub(r'\[REDACTION_SPACE\]', "\u200B", text) + text = re.sub(r'\[REDACTION_WORD ([^\]]+)\]', lambda x: repeat_along('█', x), text) + text = re.sub(r'\[ENDASH\]', '--', text) + text = re.sub(r'\[EMDASH\]', '---', text) + text = re.sub(r'\[HYPHEN\]', '-', text) + text = re.sub(r'\[CHECKBOX\]', '____', text) + text = re.sub(r'\[BLANK\]', r'__________________', text) + text = re.sub(r'\[BLANKFILL\]', r'__________________', text) + text = re.sub(r'\[PAGEBREAK\] *', '', text) + text = re.sub(r'\[PAGENUM\] *', '', text) + text = re.sub(r'\[TOTALPAGES\] *', '', text) + text = re.sub(r'\[SECTIONNUM\] *', '', text) + text = re.sub(r'\[SKIPLINE\] *', '\n\n', text) + text = re.sub(r'\[VERTICALSPACE\] *', '\n\n', text) + text = re.sub(r'\[NEWLINE\] *', '\n\n', text) + text = re.sub(r'\[NEWPAR\] *', '\n\n', text) + text = re.sub(r'\[BR\] *', '\n\n', text) + text = re.sub(r'\[TAB\] *', '', text) + text = re.sub(r' *\[END\] *', r'\n', text) + text = re.sub(r'\[BORDER\] *(.+?)\n *\n', r'\1\n\n', text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\[NOINDENT\] *(.+?)\n *\n', r'\1\n\n', text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\[FLUSHLEFT\] *(.+?)\n *\n', r'\1\n\n', text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\[FLUSHRIGHT\] *(.+?)\n *\n', r'\1\n\n', text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\[CENTER\] *(.+?)\n *\n', r'\1\n\n', text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\[BOLDCENTER\] *(.+?)\n *\n', r'**\1**\n\n', text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+)\] *(.+?)\n *\n', r'\2', text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+) *([0-9]+ *[A-Za-z]+)\] *(.+?)\n *\n', r'\3', text, flags=re.MULTILINE | re.DOTALL) + return text + + +def docx_template_filter(text, question=None, replace_newlines=True): + # logmessage('docx_template_filter') + if text == 'True': + return True + if text == 'False': + return False + if text == 'None': + return None + text = re.sub(r'\[\[([^\]]*)\]\]', r'\1', text) + text = re.sub(r'\[EMOJI ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_include_docx_template(x, question=question), text) + text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', lambda x: image_include_docx_template(x, question=question), text) + text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_include_docx_template(x, question=question), text) + text = re.sub(r'\[FILE ([^,\]]+)\]', lambda x: image_include_docx_template(x, question=question), text) + text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', qr_include_docx_template, text) + text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+)\]', qr_include_docx_template, text) + text = re.sub(r'\[QR ([^\]]+)\]', qr_include_docx_template, text) + text = re.sub(r'\[MAP ([^\]]+)\]', '', text) + text = replace_fields(text) + # text = re.sub(r'\[FIELD ([^\]]+)\]', '', text) + text = re.sub(r'\[TARGET ([^\]]+)\]', '', text) + text = re.sub(r'\[YOUTUBE[^ ]* ([^\]]+)\]', '', text) + text = re.sub(r'\[VIMEO[^ ]* ([^\]]+)\]', '', text) + text = re.sub(r'\\clearpage *\\clearpage', '', text) + text = re.sub(r'\[START_INDENTATION\]', '', text) + text = re.sub(r'\[STOP_INDENTATION\]', '', text) + text = re.sub(r'\[BEGIN_CAPTION\](.+?)\[VERTICAL_LINE\]\s*(.+?)\[END_CAPTION\]', '', text, flags=re.DOTALL) + text = re.sub(r'\[BEGIN_TWOCOL\](.+?)\[BREAK\]\s*(.+?)\[END_TWOCOL\]', '', text, flags=re.DOTALL) + text = re.sub(r'\[TIGHTSPACING\] *', '', text) + text = re.sub(r'\[SINGLESPACING\] *', '', text) + text = re.sub(r'\[DOUBLESPACING\] *', '', text) + text = re.sub(r'\[ONEANDAHALFSPACING\] *', '', text) + text = re.sub(r'\[TRIPLESPACING\] *', '', text) + text = re.sub(r'\[NBSP\]', ' ', text) + text = re.sub(r'\[REDACTION_SPACE\]', "\u200B", text) + # text = re.sub(r'\[REDACTION_SPACE\]', r'', text) + text = re.sub(r'\[REDACTION_WORD ([^\]]+)\]', lambda x: repeat_along('█', x), text) + # text = re.sub(r'\[REDACTION_WORD ([^\]]+)\]', lambda x: repeat_along('X', x), text) + text = re.sub(r'\[ENDASH\]', '--', text) + text = re.sub(r'\[EMDASH\]', '---', text) + text = re.sub(r'\[HYPHEN\]', '-', text) + text = re.sub(r'\\', '', text) + text = re.sub(r'\[CHECKBOX\]', '____', text) + text = re.sub(r'\[BLANK\]', r'__________________', text) + text = re.sub(r'\[BLANKFILL\]', r'__________________', text) + text = re.sub(r'\[PAGEBREAK\] *', '', text) + text = re.sub(r'\[PAGENUM\] *', '', text) + text = re.sub(r'\[TOTALPAGES\] *', '', text) + text = re.sub(r'\[SECTIONNUM\] *', '', text) + text = re.sub(r'\[SKIPLINE\] *', '', text) + text = re.sub(r'\[VERTICALSPACE\] *', '', text) + text = re.sub(r'\[NEWLINE\] *', '', text) + # text = re.sub(r'\n *\n', '[NEWPAR]', text) + if replace_newlines: + text = re.sub(r'\n', ' ', text) + text = re.sub(r'\[NEWPAR\] *', '', text) + text = re.sub(r'\[TAB\] *', '\t', text) + text = re.sub(r'\[NEWPAR\]', '', text) + text = re.sub(r' *\[END\] *', r'', text) + text = re.sub(r'\[BORDER\] *', r'', text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\[NOINDENT\] *', r'', text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\[FLUSHLEFT\] *', r'', text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\[FLUSHRIGHT\] *', r'', text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\[CENTER\] *', r'', text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\[BOLDCENTER\] *', r'', text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+)\] *(.+?)\n *\n', r'\2', text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+) *([0-9]+ *[A-Za-z]+)\] *(.+?)\n *\n', r'\3', text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\[BR\]', '', text) + text = re.sub(r'\[SKIPLINE\]', '', text) + text = re.sub(r'{([{%#])', '{' + zerowidth + r'\1', re.sub(r'([}%#])}', r'\1' + zerowidth + '}', text)) + return text + + +def image_include_docx(match, question=None): + file_reference = match.group(1) + if question and file_reference in question.interview.images: + file_reference = question.interview.images[file_reference].get_reference() + try: + width = match.group(2) + assert width != 'None' + width = re.sub(r'^(.*)px', convert_pixels, width) + if width == "full": + width = '100%' + except: + width = get_default_image_width() + if match.lastindex == 3: + alt_text = match.group(3) + else: + alt_text = None + if not alt_text: + alt_text = '' + file_info = file_finder(file_reference, question=question) + if 'mimetype' in file_info and file_info['mimetype']: + if re.search(r'^(audio|video)', file_info['mimetype']): + return '[reference to file type that cannot be displayed]' + if 'path' in file_info: + if 'extension' in file_info: + convert_svg_to_eps(file_info) + if file_info['extension'] in ('docx', 'rtf', 'doc', 'odt'): + if not os.path.isfile(file_info['path'] + '.pdf'): + fg_make_pdf_for_word_path(file_info['path'], file_info['extension']) + output = '![' + alt_text + '](' + file_info['path'] + '.pdf){width=' + width + '}' + return output + if file_info['extension'] in ['png', 'jpg', 'gif', 'pdf', 'eps', 'jpe', 'jpeg']: + output = '![' + alt_text + '](' + file_info['fullpath'] + '){width=' + width + '}' + return output + return '[invalid graphics reference]' + + +def image_include_docx_template(match, question=None): + file_reference = match.group(1) + if question and file_reference in question.interview.images: + file_reference = question.interview.images[file_reference].get_reference() + try: + width = match.group(2) + assert width != 'None' + width = re.sub(r'^(.*)px', convert_pixels, width) + if width == "full": + width = '100%' + except: + width = get_default_image_width() + if match.lastindex == 3: + alt_text = match.group(3) + else: + alt_text = None + file_info = file_finder(file_reference, question=question) + if 'mimetype' in file_info and file_info['mimetype']: + if re.search(r'^(audio|video)', file_info['mimetype']): + return '[reference to file type that cannot be displayed]' + if 'path' in file_info: + convert_svg_to_eps(file_info) + if 'mimetype' in file_info and file_info['mimetype']: + if file_info['mimetype'] in ('text/markdown', 'text/plain'): + with open(file_info['fullpath'], 'r', encoding='utf-8') as f: + contents = f.read() + if file_info['mimetype'] == 'text/plain': + return contents + return markdown_to_docx(contents, question, this_thread.misc.get('docx_template', None)) + if file_info['mimetype'] == 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': + return str(include_docx_template(DALocalFile(file_info['fullpath']))) + return str(image_for_docx(file_reference, question, this_thread.misc.get('docx_template', None), width=width, alt_text=alt_text)) + return '[reference to file that could not be found]' + + +def qr_include_docx(match): + the_string = match.group(1) + try: + width = match.group(2) + assert width != 'None' + width = re.sub(r'^(.*)px', convert_pixels, width) + if width == "full": + width = '100%' + except: + width = get_default_image_width() + if match.lastindex == 3: + alt_text = match.group(3) + else: + alt_text = None + if not alt_text: + alt_text = '' + im = qrcode.make(the_string) + with tempfile.NamedTemporaryFile(prefix="datemp", suffix=".png", delete=False) as the_image: + # this_thread.temporary_resources.add(the_image.name) + im.save(the_image.name) + output = '![' + alt_text + '](' + the_image.name + '){width=' + width + '}' + return output + + +def qr_include_docx_template(match): + the_string = match.group(1) + try: + width = match.group(2) + assert width != 'None' + width = re.sub(r'^(.*)px', convert_pixels, width) + if width == "full": + width = '100%' + except: + width = get_default_image_width() + if match.lastindex == 3: + alt_text = match.group(3) + else: + alt_text = None + im = qrcode.make(the_string) + with tempfile.NamedTemporaryFile(prefix="datemp", suffix=".png", delete=False) as the_image: + im.save(the_image.name) + return str(image_for_docx(DALocalFile(the_image.name), None, this_thread.misc.get('docx_template', None), width=width, alt_text=alt_text)) + + +def Alpha(number): # pylint: disable=invalid-name + multiplier = int((number - 1) / 26) + indexno = (number - 1) % 26 + return string.ascii_uppercase[indexno] * (multiplier + 1) + + +def alpha(number): + multiplier = int((number - 1) / 26) + indexno = (number - 1) % 26 + return string.ascii_lowercase[indexno] * (multiplier + 1) + + +def Roman_Numeral(number): # pylint: disable=invalid-name + return roman((number - 1) % 4000, case='upper') + + +def roman_numeral(number): + return roman((number - 1) % 4000, case='lower') + + +class SoupParser: + + def __init__(self, tpl): + self.paragraphs = [{'params': {'style': 'p', 'indentation': 0, 'list_number': 1}, 'runs': [RichText('')]}] + self.current_paragraph = self.paragraphs[-1] + self.run = self.current_paragraph['runs'][-1] + self.bold = False + self.center = False + self.list_number = 1 + self.list_type = list_types[-1] + self.italic = False + self.underline = False + self.strike = False + self.indentation = 0 + self.style = 'p' + self.still_new = True + self.size = None + self.charstyle = None + self.color = None + self.tpl = tpl + + def new_paragraph(self, classes, styles): + if self.still_new: + # logmessage("new_paragraph is still new and style is " + self.style + " and indentation is " + str(self.indentation)) + self.current_paragraph['params']['style'] = self.style + self.current_paragraph['params']['indentation'] = self.indentation + self.set_attribs(classes, styles) + self.list_number += 1 + return + # logmessage("new_paragraph where style is " + self.style + " and indentation is " + str(self.indentation)) + self.current_paragraph = {'params': {'style': self.style, 'indentation': self.indentation, 'list_number': self.list_number}, 'runs': [RichText('')]} + self.set_attribs(classes, styles) + self.list_number += 1 + self.paragraphs.append(self.current_paragraph) + self.run = self.current_paragraph['runs'][-1] + self.still_new = True + + def set_attribs(self, classes, styles): + if 'dacenter' in classes: + self.current_paragraph['params']['align'] = 'center' + elif 'daflushright' in classes: + self.current_paragraph['params']['align'] = 'end' + else: + self.current_paragraph['params']['align'] = 'start' + if len(classes): + if 'daspacingtight' in classes: + self.current_paragraph['params']['spacing'] = 240 + self.current_paragraph['params']['after'] = 0 + elif 'daspacingsingle' in classes: + self.current_paragraph['params']['spacing'] = 240 + self.current_paragraph['params']['after'] = 240 + elif 'daspacingdouble' in classes: + self.current_paragraph['params']['spacing'] = 480 + self.current_paragraph['params']['after'] = 0 + elif 'daspacingoneandahalf' in classes: + self.current_paragraph['params']['spacing'] = 260 + self.current_paragraph['params']['after'] = 0 + elif 'daspacingtriple' in classes: + self.current_paragraph['params']['spacing'] = 700 + self.current_paragraph['params']['after'] = 0 + if styles: + m = re.search(r'margin-left:([0-9\.]+)px', styles) + if m: + self.current_paragraph['params']['leftindent'] = 20 * int(m.group(1)) + m = re.search(r'margin-right:([0-9\.]+)px', styles) + if m: + self.current_paragraph['params']['rightindent'] = 20 * int(m.group(1)) + m = re.search(r'text-indent:([0-9\.]+)px', styles) + if m: + self.current_paragraph['params']['firstline'] = 20 * int(m.group(1)) + + def __str__(self): + output = '' + for para in self.paragraphs: + # logmessage("Got a paragraph where style is " + para['params'].get('style', 'undefined') + " and indentation is " + str(para['params'].get('indentation', 'undefined'))) + output += '' + if 'align' not in para['params']: + para['params']['align'] = 'start' + if para['params']['align'] == 'center': + output += '' + elif para['params']['align'] == 'end': + output += '' + if 'spacing' in para['params']: + output += '' + if para['params']['style'] == 'ul' or para['params']['style'].startswith('ol'): + if 'leftindent' in para['params']: + left_indent = para['params']['leftindent'] + else: + left_indent = 36*para['params']['indentation'] + if 'rightindent' in para['params']: + right_indent = para['params']['rightindent'] + else: + right_indent = 0 + output += '' + elif para['params']['style'] == 'blockquote': + if 'spacing' not in para['params']: + output += '' + output += '' + elif 'leftindent' in para['params'] or 'rightindent' in para['params'] or 'firstline' in para['params']: + if 'leftindent' in para['params']: + left_indent = para['params']['leftindent'] + else: + left_indent = 0 + if 'rightindent' in para['params']: + right_indent = para['params']['rightindent'] + else: + right_indent = 0 + if 'firstline' in para['params']: + first_line = para['params']['firstline'] + else: + first_line = 0 + output += '' + output += '' + if para['params']['style'] == 'ul': + output += str(RichText("•\t")) + if para['params']['style'] == 'ol1': + output += str(RichText(str(para['params']['list_number']) + ".\t")) + elif para['params']['style'] == 'olA': + output += str(RichText(Alpha(para['params']['list_number']) + ".\t")) + elif para['params']['style'] == 'ola': + output += str(RichText(alpha(para['params']['list_number']) + ".\t")) + elif para['params']['style'] == 'olI': + output += str(RichText(Roman_Numeral(para['params']['list_number']) + ".\t")) + elif para['params']['style'] == 'oli': + output += str(RichText(roman_numeral(para['params']['list_number']) + ".\t")) + for run in para['runs']: + output += str(run) + output += '' + return output + + def start_link(self, url): + ref = self.tpl.docx._part.relate_to(url, docx.opc.constants.RELATIONSHIP_TYPE.HYPERLINK, is_external=True) + self.current_paragraph['runs'].append('' % (ref, )) + self.new_run() + self.still_new = False + + def end_link(self): + self.current_paragraph['runs'].append('') + self.new_run() + self.still_new = False + + def new_run(self): + self.current_paragraph['runs'].append(RichText('')) + self.run = self.current_paragraph['runs'][-1] + + def traverse(self, elem): + for part in elem.contents: + if isinstance(part, NavigableString): + self.run.add(str(part), italic=self.italic, bold=self.bold, underline=self.underline, strike=self.strike, size=self.size, style=self.charstyle, color=self.color) + self.still_new = False + elif isinstance(part, Tag): + # logmessage("Part name is " + str(part.name)) + if part.name == 'p': + if 'class' in part.attrs: + classes = part.attrs['class'] + else: + classes = [] + if 'style' in part.attrs: + styles = part.attrs['style'] + else: + styles = "" + self.new_paragraph(classes, styles) + if 'dabold' in classes: + self.bold = True + self.traverse(part) + if 'dabold' in classes: + self.bold = False + elif part.name == 'li': + if 'class' in part.attrs: + classes = part.attrs['class'] + else: + classes = [] + if 'style' in part.attrs: + styles = part.attrs['style'] + else: + styles = "" + self.new_paragraph(classes, styles) + self.traverse(part) + elif part.name == 'ul': + # logmessage("Entering a UL") + oldstyle = self.style + self.style = 'ul' + self.indentation += 10 + self.traverse(part) + self.indentation -= 10 + self.style = oldstyle + # logmessage("Leaving a UL") + elif part.name == 'ol': + # logmessage("Entering a OL") + oldstyle = self.style + oldlistnumber = self.list_number + oldlisttype = self.list_type + if part.get('type', None) in list_types: + self.list_type = part['type'] + else: + self.list_type = list_types[(list_types.index(self.list_type) + 1) % 5] + try: + self.list_number = int(part.get('start', 1)) + except: + self.list_number = 1 + self.style = 'ol' + self.list_type + self.indentation += 10 + self.traverse(part) + self.indentation -= 10 + self.list_type = oldlisttype + self.list_number = oldlistnumber + self.style = oldstyle + # logmessage("Leaving a OL") + elif part.name == 'strong': + self.bold = True + self.traverse(part) + self.bold = False + elif part.name == 'em': + self.italic = True + self.traverse(part) + self.italic = False + elif part.name == 'strike': + self.strike = True + self.traverse(part) + self.strike = False + elif part.name == 'u': + self.underline = True + self.traverse(part) + self.underline = False + elif part.name == 'blockquote': + oldstyle = self.style + self.style = 'blockquote' + self.indentation += 20 + self.traverse(part) + self.indentation -= 20 + self.style = oldstyle + elif re.match(r'h[1-6]', part.name): + oldsize = self.size + self.size = 60 - ((int(part.name[1]) - 1) * 10) + if 'class' in part.attrs: + classes = part.attrs['class'] + else: + classes = [] + if 'style' in part.attrs: + styles = part.attrs['style'] + else: + styles = "" + self.new_paragraph(classes, styles) + self.bold = True + self.traverse(part) + self.bold = False + self.size = oldsize + elif part.name == 'a': + self.start_link(part['href']) + if self.tpl.da_hyperlink_style: + self.charstyle = self.tpl.da_hyperlink_style + else: + self.underline = True + self.color = '#0000ff' + self.traverse(part) + if self.tpl.da_hyperlink_style: + self.charstyle = None + else: + self.underline = False + self.color = None + self.end_link() + elif part.name == 'br': + self.run.add("\n", italic=self.italic, bold=self.bold, underline=self.underline, strike=self.strike, size=self.size, style=self.charstyle, color=self.color) + self.still_new = False + else: + logmessage("Encountered a " + part.__class__.__name__) + + +class InlineSoupParser: + + def __init__(self, tpl): + self.runs = [RichText('')] + self.run = self.runs[-1] + self.bold = False + self.italic = False + self.underline = False + self.indentation = 0 + self.style = 'p' + self.strike = False + self.size = None + self.charstyle = None + self.color = None + self.tpl = tpl + self.at_start = True + self.list_number = 1 + self.list_type = list_types[-1] + + def new_paragraph(self): + if self.at_start: + self.at_start = False + else: + self.run.add("\n", italic=self.italic, bold=self.bold, underline=self.underline, strike=self.strike, size=self.size, style=self.charstyle, color=self.color) + if self.indentation: + self.run.add("\t" * self.indentation) + if self.style == 'ul': + self.run.add("•\t") + if self.style == 'ol1': + self.run.add(str(self.list_number) + ".\t") + self.list_number += 1 + elif self.style == 'olA': + self.run.add(Alpha(self.list_number) + ".\t") + self.list_number += 1 + elif self.style == 'ola': + self.run.add(alpha(self.list_number) + ".\t") + self.list_number += 1 + elif self.style == 'olI': + self.run.add(Roman_Numeral(self.list_number) + ".\t") + self.list_number += 1 + elif self.style == 'oli': + self.run.add(roman_numeral(self.list_number) + ".\t") + self.list_number += 1 + # else: + # self.list_number = 1 + + def __str__(self): + output = '' + for run in self.runs: + output += str(run) + return output + + def start_link(self, url): + ref = self.tpl.docx._part.relate_to(url, docx.opc.constants.RELATIONSHIP_TYPE.HYPERLINK, is_external=True) + self.runs.append('' % (ref, )) + self.new_run() + + def end_link(self): + self.runs.append('') + self.new_run() + + def new_run(self): + self.runs.append(RichText('')) + self.run = self.runs[-1] + + def traverse(self, elem): + for part in elem.contents: + if isinstance(part, NavigableString): + self.run.add(str(part), italic=self.italic, bold=self.bold, underline=self.underline, strike=self.strike, size=self.size, style=self.charstyle, color=self.color) + elif isinstance(part, Tag): + if part.name in ('p', 'blockquote'): + self.new_paragraph() + self.traverse(part) + elif part.name == 'li': + self.new_paragraph() + self.traverse(part) + elif part.name == 'ul': + oldstyle = self.style + self.style = 'ul' + self.indentation += 1 + self.traverse(part) + self.indentation -= 1 + self.style = oldstyle + elif part.name == 'ol': + oldstyle = self.style + oldlistnumber = self.list_number + oldlisttype = self.list_type + if part.get('type', None) in list_types: + self.list_type = part['type'] + else: + self.list_type = list_types[(list_types.index(self.list_type) + 1) % 5] + try: + self.list_number = int(part.get('start', 1)) + except: + self.list_number = 1 + self.style = 'ol' + self.list_type + self.indentation += 1 + self.traverse(part) + self.indentation -= 1 + self.list_type = oldlisttype + self.list_number = oldlistnumber + self.style = oldstyle + elif part.name == 'strong': + self.bold = True + self.traverse(part) + self.bold = False + elif part.name == 'em': + self.italic = True + self.traverse(part) + self.italic = False + elif part.name == 'strike': + self.strike = True + self.traverse(part) + self.strike = False + elif part.name == 'u': + self.underline = True + self.traverse(part) + self.underline = False + elif re.match(r'h[1-6]', part.name): + oldsize = self.size + self.size = 60 - ((int(part.name[1]) - 1) * 10) + self.bold = True + self.traverse(part) + self.bold = False + self.size = oldsize + elif part.name == 'a': + self.start_link(part['href']) + if self.tpl.da_hyperlink_style: + self.charstyle = self.tpl.da_hyperlink_style + else: + self.underline = True + self.color = '#0000ff' + self.traverse(part) + if self.tpl.da_hyperlink_style: + self.charstyle = None + else: + self.underline = False + self.color = None + self.end_link() + elif part.name == 'br': + self.run.add("\n", italic=self.italic, bold=self.bold, underline=self.underline, strike=self.strike, size=self.size, style=self.charstyle, color=self.color) + else: + logmessage("Encountered a " + part.__class__.__name__) + + +def inline_markdown_to_docx(text, question, tpl): + old_context = this_thread.evaluation_context + this_thread.evaluation_context = None + try: + text = str(text) + except: + this_thread.evaluation_context = old_context + raise + this_thread.evaluation_context = old_context + source_code = markdown_to_html(text, do_terms=False) + source_code = re.sub(r"\n", ' ', source_code) + source_code = re.sub(r">\s+<", '><', source_code) + soup = BeautifulSoup('' + source_code + '', 'html.parser') + parser = InlineSoupParser(tpl) + for elem in soup.find_all(recursive=False): + parser.traverse(elem) + output = str(parser) + return docx_template_filter(output, question=question, replace_newlines=False) + + +def markdown_to_docx(text, question, tpl): + old_context = this_thread.evaluation_context + this_thread.evaluation_context = None + try: + text = str(text) + except: + this_thread.evaluation_context = old_context + raise + this_thread.evaluation_context = old_context + if daconfig.get('new markdown to docx', False): + source_code = markdown_to_html(text, do_terms=False) + source_code = re.sub(r"\n", ' ', source_code) + source_code = re.sub(r">\s+<", '><', source_code) + soup = BeautifulSoup('' + source_code + '', 'html.parser') + parser = SoupParser(tpl) + for elem in soup.find_all(recursive=False): + parser.traverse(elem) + output = str(parser) + # logmessage(output) + return docx_template_filter(output, question=question) + return inline_markdown_to_docx(text, question, tpl) + + +def include_docx_template(template_file, **kwargs): + """Include the contents of one docx file inside another docx file.""" + use_jinja = kwargs.pop('_use_jinja2', True) + if this_thread.evaluation_context is None: + return 'ERROR: not in a docx file' + if template_file.__class__.__name__ in ('DAFile', 'DAFileList', 'DAFileCollection', 'DALocalFile', 'DAStaticFile'): + template_path = template_file.path() + else: + template_path = package_template_filename(template_file, package=this_thread.current_package) + sd = this_thread.misc['docx_template'].new_subdoc() + sd.subdocx = docx.Document(template_path) + change_numbering = bool(kwargs.pop('change_numbering', True)) + if '_inline' in kwargs: + single_paragraph = True + del kwargs['_inline'] + else: + single_paragraph = False + + # We need to keep a copy of the subdocs so we can fix up the master template in the end (in parse.py) + # Given we're half way through processing the template, we can't fix the master template here + # we have to do it in post + if 'docx_subdocs' not in this_thread.misc: + this_thread.misc['docx_subdocs'] = [] + this_thread.misc['docx_subdocs'].append({'subdoc': deepcopy(sd.subdocx), 'change_numbering': change_numbering}) + + # Fix the subdocs before they are included in the template + fix_subdoc(this_thread.misc['docx_template'], {'subdoc': sd.subdocx, 'change_numbering': change_numbering}) + + first_paragraph = sd.subdocx.paragraphs[0] + + if not use_jinja: + if single_paragraph: + return re.sub(r']*>\s*(.*)\s*', r'\1', sanitize_xml(str(first_paragraph._p.xml)), flags=re.DOTALL) + return sanitize_xml(str(sd)) + + for key, val in kwargs.items(): + if hasattr(val, 'instanceName'): + the_repr = val.instanceName + elif isinstance(val, (int, float, bool, NoneType)): + the_repr = val + else: + the_repr = '_codecs.decode(_array.array("b", "' + re.sub(r'\n', '', codecs.encode(bytearray(val, encoding='utf-8'), 'base64').decode()) + '".encode()), "base64").decode()' + first_paragraph.insert_paragraph_before(str("{%%p set %s = %s %%}" % (key, the_repr))) + if 'docx_include_count' not in this_thread.misc: + this_thread.misc['docx_include_count'] = 0 + this_thread.misc['docx_include_count'] += 1 + if single_paragraph: + return re.sub(r']*>\s*(.*)\s*', r'\1', str(first_paragraph._p.xml), flags=re.DOTALL) + return sd diff --git a/docassemble_base/docassemble/base/filter/docx_subdoc.py b/docassemble_base/docassemble/base/filter/docx_subdoc.py new file mode 100644 index 000000000..bb75244fd --- /dev/null +++ b/docassemble_base/docassemble/base/filter/docx_subdoc.py @@ -0,0 +1,39 @@ +from docx.oxml.section import CT_SectPr +from docx.oxml.table import CT_Tbl +from docxcompose.composer import Composer + +def fix_subdoc(masterdoc, subdoc_info): + """Fix the images, styles, references, shapes, etc of a subdoc""" + for section in masterdoc.sections: + for part in section.part.package.parts: + if part.content_type == "application/vnd.openxmlformats-officedocument.wordprocessingml.footnotes+xml" and not isinstance(part._blob, bytes): + part._blob = part._blob.encode('utf-8') + subdoc = subdoc_info['subdoc'] + change_numbering = subdoc_info['change_numbering'] + composer = Composer(masterdoc) # Using docxcompose + composer.reset_reference_mapping() + + # This is the same as the docxcompose function, except it doesn't copy the elements over. + # Copying the elements over is done by returning the subdoc XML in this function. + # Both sd.subdocx and the master template file are changed with these functions. + composer._create_style_id_mapping(subdoc) + for element in subdoc.element.body: + if isinstance(element, CT_SectPr): + continue + composer.add_referenced_parts(subdoc.part, masterdoc.part, element) + composer.add_styles(subdoc, element) + if change_numbering and not isinstance(element, CT_Tbl): + try: + composer.add_numberings(subdoc, element) + composer.restart_first_numbering(subdoc, element) + except: + pass + composer.add_images(subdoc, element) + composer.add_shapes(subdoc, element) + composer.add_footnotes(subdoc, element) + composer.remove_header_and_footer_references(subdoc, element) + + composer.add_styles_from_other_parts(subdoc) + composer.renumber_bookmarks() + composer.renumber_docpr_ids() + composer.fix_section_types(subdoc) diff --git a/docassemble_base/docassemble/base/filter/html.py b/docassemble_base/docassemble/base/filter/html.py new file mode 100644 index 000000000..b3c22a3d1 --- /dev/null +++ b/docassemble_base/docassemble/base/filter/html.py @@ -0,0 +1,959 @@ +import re +import os +import mimetypes +import codecs +import json +from io import BytesIO +import xml.etree.ElementTree as ET +import qrcode +import qrcode.image.svg +from pikepdf import Pdf +import PIL +from bs4 import BeautifulSoup +from ..functions import get_config +from ..hooks import ( + file_finder, + get_default_thead_class, + fg_make_png_for_pdf_path, + url_finder, + get_saved_file_class, + get_default_table_class, + fg_make_pdf_for_word_path, +) +from ..language.control import get_language +from ..language.words import word +from ..thread_context import this_thread +from .utils import convert_length, replace_fields, repeat_along + +QPDF_PATH = 'qpdf' +NoneType = type(None) + +term_start = re.compile(r'\[\[') +term_match = re.compile(r'\[\[([^\[\]\|]*)(\|[^\[\]]*)?\]\]', re.DOTALL) +noquote_match = re.compile(r'"') +lt_match = re.compile(r'<') +gt_match = re.compile(r'>') +amp_match = re.compile(r'&') +# amp_match = re.compile(r'&(?!#?[0-9A-Za-z]+;)') +emoji_match = re.compile(r':([A-Za-z][A-Za-z0-9\_\-]+):') +extension_match = re.compile(r'\.[a-z]+$') +map_match = re.compile(r'\[MAP ([^\]]+)\]', flags=re.DOTALL) +code_match = re.compile(r'') + +# def blank_da_send_mail(*args, **kwargs): +# logmessage("da_send_mail: no mail agent configured!") +# return(None) + +# da_send_mail = blank_da_send_mail + +# def set_da_send_mail(func): +# global da_send_mail +# da_send_mail = func +# return + +# def blank_file_finder(*args, **kwargs): +# return({'filename': "invalid"}) + +# file_finder = blank_file_finder + +# def set_file_finder(func): +# global file_finder +# #logmessage("set the file finder to " + str(func)) +# file_finder = func +# return + +# def blank_url_finder(*args, **kwargs): +# return('about:blank') + +# url_finder = blank_url_finder + +# def set_url_finder(func): +# global url_finder +# url_finder = func +# return + +# def blank_url_for(*args, **kwargs): +# return('about:blank') + +# url_for = blank_url_for + +# def set_url_for(func): +# global url_for +# url_for = func +# return + + + +def html_filter(text, status=None, question=None, embedder=None, default_image_width=None, external=False): + if question is None and status is not None: + question = status.question + text = text + "\n\n" + text = re.sub(r'^[|] (.*)$', r'\1
', text, flags=re.MULTILINE) + text = replace_fields(text, status=status, embedder=embedder) + # if embedder is not None: + # text = re.sub(r'\[FIELD ([^\]]+)\]', lambda x: embedder(status, x.group(1)), text) + # else: + # text = re.sub(r'\[FIELD ([^\]]+)\]', 'ERROR: FIELD cannot be used here', text) + text = re.sub(r'\[TARGET ([^\]]+)\]', target_html, text) + if this_thread.evaluation_context != 'docx': + text = re.sub(r'\[EMOJI ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_url_string(x, emoji=True, question=question, external=external, status=status), text) + text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', lambda x: image_url_string(x, question=question, external=external, status=status), text) + text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_url_string(x, question=question, external=external, status=status), text) + text = re.sub(r'\[FILE ([^,\]]+)\]', lambda x: image_url_string(x, question=question, default_image_width=default_image_width, external=external, status=status), text) + text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', qr_url_string, text) + text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+)\]', qr_url_string, text) + text = re.sub(r'\[QR ([^,\]]+)\]', qr_url_string, text) + if map_match.search(text): + text = map_match.sub((lambda x: map_string(x.group(1), status)), text) + # width="420" height="315" + text = re.sub(r'\[YOUTUBE ([^\]]+)\]', r'
', text) + text = re.sub(r'\[YOUTUBE4:3 ([^\]]+)\]', r'
', text) + text = re.sub(r'\[YOUTUBE16:9 ([^\]]+)\]', r'
', text) + # width="500" height="281" + text = re.sub(r'\[VIMEO ([^\]]+)\]', r'
', text) + text = re.sub(r'\[VIMEO4:3 ([^\]]+)\]', r'
', text) + text = re.sub(r'\[VIMEO16:9 ([^\]]+)\]', r'
', text) + text = re.sub(r'\[BEGIN_CAPTION\](.+?)\[VERTICAL_LINE\]\s*(.+?)\[END_CAPTION\]', html_caption, text, flags=re.DOTALL) + text = re.sub(r'\[BEGIN_TWOCOL\](.+?)\[BREAK\]\s*(.+?)\[END_TWOCOL\]', html_two_col, text, flags=re.DOTALL) + text = re.sub(r'\[NBSP\]', r' ', text) + text = re.sub(r'\[REDACTION_SPACE\]', '█​', text) + text = re.sub(r'\[REDACTION_WORD ([^\]]+)\]', lambda x: repeat_along('█', x), text) + text = re.sub(r'\[ENDASH\]', r'–', text) + text = re.sub(r'\[EMDASH\]', r'—', text) + text = re.sub(r'\[HYPHEN\]', r'-', text) + text = re.sub(r'\[CHECKBOX\]', r'    ', text) + text = re.sub(r'\[BLANK\]', r'            ', text) + text = re.sub(r'\[BLANKFILL\]', r'                              ', text) + text = re.sub(r'\[PAGEBREAK\] *', r'', text) + text = re.sub(r'\[PAGENUM\] *', r'', text) + text = re.sub(r'\[SECTIONNUM\] *', r'', text) + text = re.sub(r'\[SKIPLINE\] *', r'
', text) + text = re.sub(r'\[NEWLINE\] *', r'
', text) + text = re.sub(r'\[NEWPAR\] *', r'

', text) + text = re.sub(r'\[BR\] *', r'
', text) + text = re.sub(r'\[TAB\] *', '', text) + text = re.sub(r' *\[END\] *', r'\n', text) + lines = re.split(r'\n *\n', text) + text = '' + spacing_class = None + doing_indentation = False + for line in lines: + classes = set() + styles = {} + if re.search(r'\[TIGHTSPACING\]', line): + spacing_class = 'daspacingtight' + if re.search(r'\[SINGLESPACING\]', line): + spacing_class = 'daspacingsingle' + if re.search(r'\[DOUBLESPACING\]', line): + spacing_class = 'daspacingdouble' + if re.search(r'\[ONEANDAHALFSPACING\]', line): + spacing_class = 'daspacingoneandahalf' + if re.search(r'\[TRIPLESPACING\]', line): + spacing_class = 'daspacingtriple' + if re.search(r'\[START_INDENTATION\]', line): + doing_indentation = True + if re.search(r'\[STOP_INDENTATION\]', line): + doing_indentation = False + if spacing_class: + classes.add(spacing_class) + if doing_indentation and not re.search(r'\[NOINDENT\]', line): + styles['text-indent'] = '36px' + if re.search(r'\[BORDER\]', line): + classes.add('daborder') + if re.search(r'\[FLUSHLEFT\]', line): + classes.add('daflushleft') + if re.search(r'\[FLUSHRIGHT\]', line): + classes.add('daflushright') + if re.search(r'\[CENTER\]', line): + classes.add('dacenter') + if re.search(r'\[BOLDCENTER\]', line): + classes.add('dacenter') + classes.add('dabold') + m = re.search(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+)\]', line) + if m: + styles["padding-left"] = str(convert_length(m.group(1), 'px')) + 'px' + m = re.search(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+) *([0-9]+ *[A-Za-z]+)\]', line) + if m: + styles["margin-left"] = str(convert_length(m.group(1), 'px')) + 'px' + styles["margin-right"] = str(convert_length(m.group(2), 'px')) + 'px' + orig_length = len(line) + line = re.sub(r'\[(BORDER|NOINDENT|FLUSHLEFT|FLUSHRIGHT|BOLDCENTER|CENTER|TIGHTSPACING|SINGLESPACING|DOUBLESPACING|ONEANDAHALFSPACING|TRIPLESPACING|START_INDENTATION|STOP_INDENTATION)\] *', r'', line) + line = re.sub(r'\[INDENTBY[^\]]*\] *', r'', line) + if orig_length > 0 and len(line) == 0: + continue + if line.startswith('>'): + line = re.sub(r'^> *', '', line) + text += "> " + if len(classes) > 0 or len(styles) > 0: + text += ' 0: + text += ' style="' + "".join(map(lambda x: str(x[0]) + ":" + x[1] + ';', styles.items())) + '"' + text += '>' + text += line + '\n\n' + text = re.sub(r'\n+$', r'', text) + return text + + +def map_string(encoded_text, status): + if status is None: + return '' + map_number = len(status.maps) + status.maps.append(codecs.decode(bytearray(encoded_text, 'utf-8'), 'base64').decode()) + return '
' + + +def target_html(match): + target = match.group(1) + target = re.sub(r'[^A-Za-z0-9\_]', r'', str(target)) + return '' + + +def html_caption(match): + firstcol = match.group(1) + secondcol = match.group(2) + firstcol = re.sub(r'^\s+', '', firstcol) + firstcol = re.sub(r'\s+$', '', firstcol) + secondcol = re.sub(r'^\s+', '', secondcol) + secondcol = re.sub(r'\s+$', '', secondcol) + firstcol = markdown_to_html(firstcol) + secondcol = markdown_to_html(secondcol) + return '
' + firstcol + '' + secondcol + '
' + + +def html_two_col(match): + firstcol = markdown_to_html(match.group(1)) + secondcol = markdown_to_html(match.group(2)) + return '
' + firstcol + '' + secondcol + '
' + + +def add_newlines(string): + string = re.sub(r'\[(BR)\]', r'[NEWLINE]', string) + string = re.sub(r' *\n', r'\n', string) + string = re.sub(r'(? 0: + return audio_control(urls) + return '' + if re.search(r'^video', file_info['mimetype']): + urls = get_video_urls([{'text': "[FILE " + file_reference + "]", 'package': None, 'type': 'video'}], question=question) + if len(urls) > 0: + return video_control(urls) + return '' + if 'extension' in file_info and file_info['extension'] is not None: + if re.match(r'.*%$', width): + width_string = "width:" + width + stack_width_string = width_string + else: + width_string = "max-width:" + width + stack_width_string = "width:" + width + if emoji: + width_string += ';vertical-align: middle' + alt_text = 'alt="" ' + the_url = url_finder(file_reference, _question=question, display_filename=file_info['filename'], _external=external) + if the_url is None: + return '[ERROR: File reference ' + str(file_reference) + ' cannot be displayed]' + if width_string == 'width:100%': + extra_class = ' dawideimage' + else: + extra_class = '' + if file_info.get('extension', '') in ('png', 'jpg', 'gif', 'svg', 'jpe', 'jpeg'): + try: + if file_info.get('extension', '') == 'svg': + attributes = ET.parse(file_info['fullpath']).getroot().attrib + layout_width = attributes['width'] + layout_height = attributes['height'] + else: + with PIL.Image.open(file_info['fullpath']) as im: + layout_width, layout_height = im.size + return '' + except: + return '' + if file_info['extension'] in ('pdf', 'docx', 'rtf', 'doc', 'odt'): + if file_info['extension'] in ('docx', 'rtf', 'doc', 'odt') and not os.path.isfile(file_info['path'] + '.pdf'): + fg_make_pdf_for_word_path(file_info['path'], file_info['extension']) + fg_make_png_for_pdf_path(file_info['path'] + ".pdf", 'screen', page=1) + if re.match(r'[0-9]+', str(file_reference)): + sf = get_saved_file_class()(int(file_reference), fix=True) + sf.finalize() + if 'pages' not in file_info: + try: + with Pdf.open(file_info['path'] + '.pdf') as reader: + file_info['pages'] = len(reader.pages) + except: + file_info['pages'] = 1 + the_image_url = url_finder(file_reference, size="screen", page=1, _question=question, _external=external) + if the_image_url is None: + return '[ERROR: File reference ' + str(file_reference) + ' cannot be displayed]' + if 'filename' in file_info: + title = ' title="' + file_info['filename'] + if 'pages' in file_info and file_info['pages'] > 1: + title += " (" + str(file_info['pages']) + " " + word('pages') + ")" + title += '"' + else: + if 'pages' in file_info and file_info['pages'] > 1: + title = ' title="' + str(file_info['pages']) + " " + word('pages') + '"' + else: + title = '' + if alt_text == '': + the_alt_text = 'alt=' + json.dumps(word("Thumbnail image of document")) + ' ' + else: + the_alt_text = alt_text + try: + with Pdf.open(file_info['path'] + '.pdf') as reader: + layout_width = reader.pages[0].mediabox[2] - reader.pages[0].mediabox[0] + layout_height = reader.pages[0].mediabox[3] - reader.pages[0].mediabox[1] + if width_string == 'width:100%': + output = '' + else: + if 'pages' in file_info and file_info['pages'] >= 1: + extra_pages = min(2, file_info['pages'] - 1) + else: + extra_pages = 2 + aspect_ratio = 1.0*layout_width/layout_height + stack_width_string += "; height: auto; aspect-ratio: " + str(aspect_ratio) + ";" + output = '
' + (('
') * extra_pages) + '
' + except: + output = '' + return output + return '' + file_info['filename'] + '' + return '[Invalid image reference; reference=' + str(file_reference) + ', width=' + str(width) + ', filename=' + file_info.get('filename', 'unknown') + ']' + + +def qr_url_string(match): + string = match.group(1) + try: + width = match.group(2) + assert width != 'None' + except: + width = "300px" + if width == "full": + width = "300px" + if match.lastindex == 3: + if match.group(3) != 'None': + alt_text = str(match.group(3)) + else: + alt_text = word(f"A QR code that goes to {string}") + else: + alt_text = word(f"A QR code that goes to {string}") + width_string = "width:" + width + im = qrcode.make(string, image_factory=qrcode.image.svg.SvgPathFillImage) + output = BytesIO() + im.save(output) + the_image = output.getvalue().decode() + the_image = re.sub(r"<\?xml version='1.0' encoding='UTF-8'\?>\n", '', the_image) + the_image = re.sub(r'height="[0-9]+mm" ', '', the_image) + the_image = re.sub(r'width="[0-9]+mm" ', '', the_image) + m = re.search(r'(viewBox="[^"]+")', the_image) + if m: + viewbox = m.group(1) + else: + viewbox = "" + return '' + the_image + '' + alt_text + '' + + +def get_icon_html(text): + icons_setting = get_config('default icons', None) + if icons_setting == 'font awesome': + m = re.search(r'^(fa[a-z])-fa-(.*)', text) + if m: + the_prefix = m.group(1) + text = m.group(2) + else: + the_prefix = get_config('font awesome prefix', 'fa-solid') + if the_prefix == 'fab': + the_prefix = 'fa-brands' + elif the_prefix == 'far': + the_prefix = 'fa-regular' + elif the_prefix == 'fas': + the_prefix = 'fa-solid' + return '' + if icons_setting == 'material icons': + return '' + str(text) + '' + return None + + +def emoji_html(text, status=None, question=None, images=None): + # logmessage("Got to emoji_html") + if status is not None and question is None: + question = status.question + if images is None: + images = question.interview.images + if text in images: + if status is not None and images[text].attribution is not None: + status.attributions.add(images[text].attribution) + return image_url(images[text].get_reference(), word('icon'), '1em', emoji=True, question=question) + icon_html = get_icon_html(text) + if icon_html: + return icon_html + return ":" + str(text) + ":" + + +def emoji_insert(text, status=None, images=None): + if images is None: + images = status.question.interview.images + if text in images: + if status is not None and images[text].attribution is not None: + status.attributions.add(images[text].attribution) + return "[EMOJI " + images[text].get_reference() + ', 1.2em]' + return ":" + str(text) + ":" + + +def link_rewriter(m, status): + the_path = None + if m.group(1).startswith('#'): + return ' 0: + lang = get_language() + for term in question.terms: + terms_done.add(term.lower()) + # logmessage("Searching for term " + term + " in " + a) + if lang in question.terms[term]['re']: + a = question.terms[term]['re'][lang].sub(sub_term, a) + else: + a = question.terms[term]['re'][question.language].sub(sub_term, a) + # logmessage("string is now " + str(a)) + if len(question.autoterms) > 0: + lang = get_language() + for term in question.autoterms: + if term.lower() in terms_done: + continue + terms_done.add(term.lower()) + # logmessage("Searching for term " + term + " in " + a) + if lang in question.autoterms[term]['re']: + a = question.autoterms[term]['re'][lang].sub(r'[[\1]]', a) + else: + a = question.autoterms[term]['re'][question.language].sub(r'[[\1]]', a) + # logmessage("string is now " + str(a)) + if 'interview_terms' in status.extras: + interview_terms = status.extras['interview_terms'] + else: + interview_terms = question.interview.terms + if 'interview_autoterms' in status.extras: + interview_autoterms = status.extras['interview_autoterms'] + else: + interview_autoterms = question.interview.autoterms + else: + interview_terms = question.interview.terms + interview_autoterms = question.interview.autoterms + if len(interview_terms) > 0: + lang = get_language() + if lang in interview_terms and len(interview_terms[lang]) > 0: + for term in interview_terms[lang]: + if term.lower() in terms_done: + continue + terms_done.add(term.lower()) + # logmessage("Searching for term " + term + " in " + a) + a = interview_terms[lang][term]['re'].sub(sub_term, a) + # logmessage("string is now " + str(a)) + elif question.language in interview_terms and len(interview_terms[question.language]) > 0: + for term in interview_terms[question.language]: + if term.lower() in terms_done: + continue + terms_done.add(term.lower()) + # logmessage("Searching for term " + term + " in " + a) + a = interview_terms[question.language][term]['re'].sub(sub_term, a) + # logmessage("string is now " + str(a)) + if len(interview_autoterms) > 0: + lang = get_language() + if lang in interview_autoterms and len(interview_autoterms[lang]) > 0: + for term in interview_autoterms[lang]: + if term.lower() in terms_done: + continue + terms_done.add(term.lower()) + # logmessage("Searching for term " + term + " in " + a) + a = interview_autoterms[lang][term]['re'].sub(r'[[\1]]', a) + # logmessage("string is now " + str(a)) + elif question.language in interview_autoterms and len(interview_autoterms[question.language]) > 0: + for term in interview_autoterms[question.language]: + if term.lower() in terms_done: + continue + terms_done.add(term.lower()) + # logmessage("Searching for term " + term + " in " + a) + a = interview_autoterms[question.language][term]['re'].sub(r'[[\1]]', a) + # logmessage("string is now " + str(a)) + a = html_filter(str(a), status=status, question=question, embedder=embedder, default_image_width=default_image_width, external=external) + # logmessage("before: " + a) + if status and status.extras.get('tableCssClass', None): + classes = status.extras['tableCssClass'].split(',') + table_class = json.dumps(classes[0].strip()) + if len(classes) > 1: + thead_class = json.dumps(classes[1].strip()) + else: + thead_class = None + else: + table_class = get_default_table_class() + thead_class = get_default_thead_class() + a = re.sub(r'<(/?)table', r'<\1TABLE', a) + a = re.sub(r'', r'', a) + if use_pandoc: + from docassemble.base import pandoc + converter = pandoc.MyPandoc() + converter.output_format = 'html' + converter.input_content = a + converter.convert(question) + result = converter.output_content + else: + try: + result = this_thread.markdown.reset().convert(a) + except: + # Try again because sometimes it fails randomly and maybe trying again will work. + result = this_thread.markdown.reset().convert(a) + result = re.sub(r'', r'
', result) + if thead_class != '': + result = re.sub(r'', r'', result) + result = re.sub(r'
', r'', result) + result = re.sub(r'<(/?)TABLE', r'<\1table', result) + result = re.sub(r'', r'', result) + result = re.sub(r'<(t[dh]) align="(right|left|center)">', r'<\1 class="text-\2">', result) + result = re.sub(r'
', r'
', result) + result = re.sub(r' 0 and 'terms' in status.extras: + result = term_match.sub((lambda x: add_terms(x.group(1), status.extras['terms'], label=x.group(2), status=status, question=question)), result) + if len(question.autoterms) > 0 and 'autoterms' in status.extras: + result = term_match.sub((lambda x: add_terms(x.group(1), status.extras['autoterms'], label=x.group(2), status=status, question=question)), result) + if 'interview_terms' in status.extras: + interview_terms = status.extras['interview_terms'] + else: + interview_terms = question.interview.terms + if 'interview_autoterms' in status.extras: + interview_autoterms = status.extras['interview_autoterms'] + else: + interview_autoterms = question.interview.autoterms + else: + interview_terms = question.interview.terms + interview_autoterms = question.interview.autoterms + if lang in interview_terms and len(interview_terms[lang]): + result = term_match.sub((lambda x: add_terms(x.group(1), interview_terms[lang], label=x.group(2), status=status, question=question)), result) + elif question.language in interview_terms and len(interview_terms[question.language]): + result = term_match.sub((lambda x: add_terms(x.group(1), interview_terms[question.language], label=x.group(2), status=status, question=question)), result) + if lang in interview_autoterms and len(interview_autoterms[lang]): + result = term_match.sub((lambda x: add_terms(x.group(1), interview_autoterms[lang], label=x.group(2), status=status, question=question)), result) + elif question.language in interview_autoterms and len(interview_autoterms[question.language]): + result = term_match.sub((lambda x: add_terms(x.group(1), interview_autoterms[question.language], label=x.group(2), status=status, question=question)), result) + do_not_scan_for_emojis = bool(re.search(r'\[NO_EMOJIS\]', result)) + if do_not_scan_for_emojis: + result = re.sub(r'\[NO_EMOJIS\]\s*', r'', result) + if status is not None and question.interview.scan_for_emojis and not do_not_scan_for_emojis: + result = emoji_match.sub((lambda x: emoji_html(x.group(1), status=status, question=question)), result) + result = re.sub(r'

', result) + if trim: + if result.startswith('

') and result.endswith('

'): + result = re.sub(r'

\s*

', ' ', result[3:-4]) + elif pclass: + result = re.sub('

', '

', result) + if escape: + if escape is True: + result = noquote_match.sub('"', result) + if escape == 'option': + result = re.sub(r'\n\r', ' ', BeautifulSoup(result, 'html.parser').get_text()).strip() + result = lt_match.sub('<', result) + result = gt_match.sub('>', result) + if escape is True: + result = amp_match.sub('&', result) + # logmessage("after: " + result) + # result = result.replace('\n', ' ') + if result: + if strip_newlines: + result = result.replace('\n', ' ') + if divclass is not None: + result = '

' + result + '
' + # if indent and not code_match.search(result): + # return (" " * indent) + re.sub(r'\n', "\n" + (" " * indent), result).rstrip() + "\n" + return result + + +def my_escape(result): + result = noquote_match.sub('"', result) + result = lt_match.sub('<', result) + result = gt_match.sub('>', result) + result = amp_match.sub('&', result) + return result + + +def noquote(string): + # return json.dumps(string.replace('\n', ' ').rstrip()) + return '"' + string.replace('\n', ' ').replace('"', '"').rstrip() + '"' + + +def add_terms_mako(termname, terms, status=None, question=None): + lower_termname = re.sub(r'\s+', ' ', str(termname).lower(), re.DOTALL) + if lower_termname in terms: + term_as_text = to_text(markdown_to_html(str(termname), trim=False, do_terms=False, status=status, question=question), None, None) + return '
' + str(termname) + '' + # logmessage(lower_termname + " is not in terms dictionary") + return '[[' + termname + ']]' + + +def add_terms(termname, terms, label=None, status=None, question=None): + if label is None: + label = str(termname) + else: + label = re.sub(r'^\|', '', label) + lower_termname = re.sub(r'\s+', ' ', termname.lower(), re.DOTALL) + if lower_termname in terms: + term_as_text = to_text(markdown_to_html(label, trim=False, do_terms=False, status=status, question=question), None, None) + return '' + label + '' + return '[[' + termname + ']]' + + +def audio_control(files, preload="metadata", title_text=None): + for d in files: + if isinstance(d, str): + return d + if title_text is None: + title_text = '' + else: + title_text = " title=" + json.dumps(title_text) + output = '' + "\n" + for d in files: + if isinstance(d, list): + output += ' ' + output += "\n" + output += ' ' + word('Listen') + '\n' + output += "\n" + return output + + +def video_control(files): + for d in files: + if isinstance(d, (str, NoneType)): + return str(d) + output = '\n" + return output + + +def get_audio_urls(the_audio, question=None): + output = [] + the_list = [] + to_try = {} + for audio_item in the_audio: + if audio_item['type'] != 'audio': + continue + found_upload = False + pattern = re.compile(r'^\[FILE ([^,\]]+)') + for file_ref in re.findall(pattern, audio_item['text']): + found_upload = True + m = re.match(r'[0-9]+', file_ref) + if m: + file_info = file_finder(file_ref, question=question) + if 'path' in file_info: + if file_info['mimetype'] == 'audio/ogg': + output.append([url_finder(file_ref, _question=question), file_info['mimetype']]) + elif os.path.isfile(file_info['path'] + '.ogg'): + output.append([url_finder(file_ref, ext='ogg', _question=question), 'audio/ogg']) + if file_info['mimetype'] == 'audio/mpeg': + output.append([url_finder(file_ref, _question=question), file_info['mimetype']]) + elif os.path.isfile(file_info['path'] + '.mp3'): + output.append([url_finder(file_ref, ext='mp3', _question=question), 'audio/mpeg']) + if file_info['mimetype'] not in ['audio/mpeg', 'audio/ogg']: + output.append([url_finder(file_ref, _question=question), file_info['mimetype']]) + else: + the_list.append({'text': file_ref, 'package': audio_item['package']}) + if not found_upload: + the_list.append(audio_item) + for audio_item in the_list: + mimetype, encoding = mimetypes.guess_type(audio_item['text']) # pylint: disable=unused-variable + if re.search(r'^http', audio_item['text']): + output.append([audio_item['text'], mimetype]) + continue + basename = os.path.splitext(audio_item['text'])[0] + ext = os.path.splitext(audio_item['text'])[1] + if mimetype not in to_try: + to_try[mimetype] = [] + to_try[mimetype].append({'basename': basename, 'filename': audio_item['text'], 'ext': ext, 'package': audio_item['package']}) + if 'audio/mpeg' in to_try and 'audio/ogg' not in to_try: + to_try['audio/ogg'] = [] + for attempt in to_try['audio/mpeg']: + if attempt['ext'] == '.MP3': + to_try['audio/ogg'].append({'basename': attempt['basename'], 'filename': attempt['basename'] + '.OGG', 'ext': '.OGG', 'package': attempt['package']}) + else: + to_try['audio/ogg'].append({'basename': attempt['basename'], 'filename': attempt['basename'] + '.ogg', 'ext': '.ogg', 'package': attempt['package']}) + if 'audio/ogg' in to_try and 'audio/mpeg' not in to_try: + to_try['audio/mpeg'] = [] + for attempt in to_try['audio/ogg']: + if attempt['ext'] == '.OGG': + to_try['audio/mpeg'].append({'basename': attempt['basename'], 'filename': attempt['basename'] + '.MP3', 'ext': '.MP3', 'package': attempt['package']}) + else: + to_try['audio/mpeg'].append({'basename': attempt['basename'], 'filename': attempt['basename'] + '.mp3', 'ext': '.mp3', 'package': attempt['package']}) + for mimetype in reversed(sorted(to_try.keys())): + for attempt in to_try[mimetype]: + parts = attempt['filename'].split(':') + if len(parts) < 2: + parts = [attempt['package'], attempt['filename']] + if parts[0] is None: + parts[0] = 'None' + parts[1] = re.sub(r'^data/static/', '', parts[1]) + full_file = parts[0] + ':data/static/' + parts[1] + file_info = file_finder(full_file, question=question) + if 'fullpath' in file_info: + url = url_finder(full_file, _question=question) + output.append([url, mimetype]) + return [item for item in output if item[0] is not None] + + +def get_video_urls(the_video, question=None): + output = [] + the_list = [] + to_try = {} + for video_item in the_video: + if video_item['type'] != 'video': + continue + found_upload = False + if re.search(r'^\[(YOUTUBE|VIMEO)[0-9\:]* ', video_item['text']): + output.append(html_filter(video_item['text'])) + continue + pattern = re.compile(r'^\[FILE ([^,\]]+)') + for file_ref in re.findall(pattern, video_item['text']): + found_upload = True + m = re.match(r'[0-9]+', file_ref) + if m: + file_info = file_finder(file_ref, question=question) + if 'path' in file_info: + if file_info['mimetype'] == 'video/ogg': + output.append([url_finder(file_ref, _question=question), file_info['mimetype']]) + elif os.path.isfile(file_info['path'] + '.ogv'): + output.append([url_finder(file_ref, ext='ogv', _question=question), 'video/ogg']) + if file_info['mimetype'] == 'video/mp4': + output.append([url_finder(file_ref, _question=question), file_info['mimetype']]) + elif os.path.isfile(file_info['path'] + '.mp4'): + output.append([url_finder(file_ref, ext='mp4', _question=question), 'video/mp4']) + if file_info['mimetype'] not in ['video/mp4', 'video/ogg']: + output.append([url_finder(file_ref, _question=question), file_info['mimetype']]) + else: + the_list.append({'text': file_ref, 'package': video_item['package']}) + if not found_upload: + the_list.append(video_item) + for video_item in the_list: + mimetype, encoding = mimetypes.guess_type(video_item['text']) # pylint: disable=unused-variable + if re.search(r'^http', video_item['text']): + output.append([video_item['text'], mimetype]) + continue + basename = os.path.splitext(video_item['text'])[0] + ext = os.path.splitext(video_item['text'])[1] + if mimetype not in to_try: + to_try[mimetype] = [] + to_try[mimetype].append({'basename': basename, 'filename': video_item['text'], 'ext': ext, 'package': video_item['package']}) + if 'video/mp4' in to_try and 'video/ogg' not in to_try: + to_try['video/ogg'] = [] + for attempt in to_try['video/mp4']: + if attempt['ext'] == '.MP4': + to_try['video/ogg'].append({'basename': attempt['basename'], 'filename': attempt['basename'] + '.OGV', 'ext': '.OGV', 'package': attempt['package']}) + else: + to_try['video/ogg'].append({'basename': attempt['basename'], 'filename': attempt['basename'] + '.ogv', 'ext': '.ogv', 'package': attempt['package']}) + if 'video/ogg' in to_try and 'video/mp4' not in to_try: + to_try['video/mp4'] = [] + for attempt in to_try['video/ogg']: + if attempt['ext'] == '.OGV': + to_try['video/mp4'].append({'basename': attempt['basename'], 'filename': attempt['basename'] + '.MP4', 'ext': '.MP4', 'package': attempt['package']}) + else: + to_try['audio/mpeg'].append({'basename': attempt['basename'], 'filename': attempt['basename'] + '.mp4', 'ext': '.mp4', 'package': attempt['package']}) + for mimetype in reversed(sorted(to_try.keys())): + for attempt in to_try[mimetype]: + parts = attempt['filename'].split(':') + if len(parts) < 2: + parts = [attempt['package'], attempt['filename']] + parts[1] = re.sub(r'^data/static/', '', parts[1]) + if parts[0] is None: + full_file = 'data/static/' + parts[1] + else: + full_file = parts[0] + ':data/static/' + parts[1] + file_info = file_finder(full_file, question=question) + if 'fullpath' in file_info: + url = url_finder(full_file, _question=question) + if url is not None: + output.append([url, mimetype]) + return output + + +def process_target(text): + return re.sub(r'\[TARGET ([^\]]+)\]', target_html, text) + + +def to_text(html_doc, terms, links): + output = "" + # logmessage("to_text: html doc is " + str(html_doc)) + if not html_doc.startswith('<'): + html_doc = "" + html_doc + "" + soup = BeautifulSoup(html_doc, 'html.parser') + [s.extract() for s in soup(['style', 'script', '[document]', 'head', 'title', 'audio', 'video', 'pre', 'attribution'])] # pylint: disable=expression-not-assigned + [s.extract() for s in soup.find_all(hidden)] # pylint: disable=expression-not-assigned + [s.extract() for s in soup.find_all('div', {'class': 'dainvisible'})] # pylint: disable=expression-not-assigned + for s in soup.find_all(do_show): + if s.name in ['input', 'textarea', 'img'] and s.has_attr('alt'): + words = s.attrs['alt'] + if s.has_attr('placeholder'): + words += ", " + s.attrs['placeholder'] + else: + words = s.get_text() + words = re.sub(r'\n\s*', ' ', words, flags=re.DOTALL) + output += words + "\n" + for s in soup.find_all('a'): + if s.has_attr('class') and s.attrs['class'][0] == 'daterm' and s.has_attr('data-bs-content'): + terms[s.string] = s.attrs['data-bs-content'] + elif s.has_attr('href'): # and (s.attrs['href'].startswith(url) or s.attrs['href'].startswith('?')): + # logmessage("Adding a link: " + s.attrs['href']) + links.append((s.attrs['href'], s.get_text())) + output = re.sub(br'\u201c'.decode('raw_unicode_escape'), '"', output) + output = re.sub(br'\u201d'.decode('raw_unicode_escape'), '"', output) + output = re.sub(br'\u2018'.decode('raw_unicode_escape'), "'", output) + output = re.sub(br'\u2019'.decode('raw_unicode_escape'), "'", output) + output = re.sub(br'\u201b'.decode('raw_unicode_escape'), "'", output) + output = re.sub(r'&gt;', '>', output) + output = re.sub(r'&lt;', '<', output) + output = re.sub(r'>', '>', output) + output = re.sub(r'<', '<', output) + output = re.sub(r'<[^>]+>', '', output) + output = re.sub(r'\n$', '', output) + output = re.sub(r' +', ' ', output) + return output + +bad_list = ['div', 'option'] + +good_list = ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'button', 'textarea', 'note'] + + +def do_show(element): + if re.match('', str(element), re.DOTALL): + return False + if element.name in ['option'] and element.has_attr('selected'): + return True + if element.name in bad_list: + return False + if element.name in ['img', 'input'] and element.has_attr('alt'): + return True + if element.name in good_list: + return True + if element.parent and element.parent.name in good_list: + return False + if element.string: + return True + if re.match(r'\s+', element.get_text()): + return False + return False + + +def hidden(element): + if element.name == 'input': + if element.has_attr('type'): + if element.attrs['type'] == 'hidden': + return True + return False diff --git a/docassemble_base/docassemble/base/filter/image_docx.py b/docassemble_base/docassemble/base/filter/image_docx.py new file mode 100644 index 000000000..662a7bc34 --- /dev/null +++ b/docassemble_base/docassemble/base/filter/image_docx.py @@ -0,0 +1,56 @@ +import re +from docxtpl import InlineImage +from docx.shared import Mm, Inches, Pt, Cm, Twips +from docassemble.base.hooks import file_finder +from docassemble.base.filter.utils import convert_svg_to_png + + +def fix_double_quote(the_string): + return '"' + re.sub('"', '"', the_string) + '"' + + +class CustomInlineImage(InlineImage): + alt_text = None + + def __init__(self, tpl, image_descriptor, width=None, height=None, anchor=None, alt_text=None): + super().__init__(tpl, image_descriptor, width=width, height=height, anchor=anchor) + self.alt_text = alt_text + + def _insert_image(self): + output = super()._insert_image() + if self.alt_text: + return re.sub(' 0: + prior_values = formatting_stack.pop() + spacing_command = prior_values['spacing_command'] + after_space = prior_values['after_space'] + default_indentation = prior_values['default_indentation'] + indentation_command = prior_values['indentation_command'] + elif re.search(r'\[TIGHTSPACING\]', line): + spacing_command = rtf_spacing['tight'] + default_spacing = 'tight' + after_space = after_space_multiplier * rtf_after_space[default_spacing] + default_indentation = False + elif re.search(r'\[SINGLESPACING\]', line): + spacing_command = rtf_spacing['single'] + default_spacing = 'single' + after_space = after_space_multiplier * rtf_after_space[default_spacing] + default_indentation = False + elif re.search(r'\[ONEANDAHALFSPACING\]', line): + spacing_command = rtf_spacing['oneandahalf'] + default_spacing = 'oneandahalf' + after_space = after_space_multiplier * rtf_after_space[default_spacing] + elif re.search(r'\[DOUBLESPACING\]', line): + spacing_command = rtf_spacing['double'] + default_spacing = 'double' + after_space = after_space_multiplier * rtf_after_space[default_spacing] + elif re.search(r'\[TRIPLESPACING\]', line): + spacing_command = rtf_spacing['triple'] + default_spacing = 'triple' + after_space = after_space_multiplier * rtf_after_space[default_spacing] + elif re.search(r'\[START_INDENTATION\]', line): + indentation_command = r'\\fi' + str(indentation_amount) + " " + elif re.search(r'\[STOP_INDENTATION\]', line): + indentation_command = r'\\fi0 ' + elif line != '': + special_after_space = None + special_spacing = None + if re.search(r'\[BORDER\]', line): + line = re.sub(r' *\[BORDER\] *', r'', line) + border_text = r'\\box \\brdrhair \\brdrw1 \\brdrcf1 \\brsp29 ' + else: + border_text = r'' + line = re.sub(r'{(\\pard\\intbl \\q[lrc] \\f[0-9]+ \\sa[0-9]+ \\li[0-9]+ \\fi[0-9]+.*?)\\par}', r'\1', line) + if re.search(r'\[NOPAR\]', line): + line = re.sub(r'{\\pard \\ql \\f[0-9]+ \\sa[0-9]+ \\li[0-9]+ \\fi-?[0-9]* *(.*?)\\par}', r'\1', line) + line = re.sub(r' *\[NOPAR\] *', r'', line) + n = re.search(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+) *([0-9\.]+ *[A-Za-z]+)\]', line) + m = re.search(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+)\]', line) + if n: + line = re.sub(r'\\fi-?[0-9]+ ', r'\\fi0 ', line) + line = re.sub(r'\\ri-?[0-9]+ ', r'', line) + line = re.sub(r'\\li-?[0-9]+ ', r'\\li' + str(convert_length(n.group(1), 'twips')) + r' \\ri' + str(convert_length(n.group(2), 'twips')) + ' ', line) + line = re.sub(r'\[INDENTBY[^\]]*\]', '', line) + elif m: + line = re.sub(r'\\fi-?[0-9]+ ', r'\\fi0 ', line) + line = re.sub(r'\\li-?[0-9]+ ', r'\\li' + str(convert_length(m.group(1), 'twips')) + ' ', line) + line = re.sub(r' *\[INDENTBY[^\]]*\] *', '', line) + elif re.search(r'\[NOINDENT\]', line): + line = re.sub(r'\\fi-?[0-9]+ ', r'\\fi0 ', line) + line = re.sub(r' *\[NOINDENT\] *', '', line) + elif re.search(r'\[FLUSHLEFT\]', line): + line = re.sub(r'\\fi-?[0-9]+ ', r'\\fi0 ', line) + line = re.sub(r' *\[FLUSHLEFT\] *', '', line) + special_after_space = after_space_multiplier * 1 + special_spacing = rtf_spacing['single'] + elif re.search(r'\[FLUSHRIGHT\]', line): + line = re.sub(r'\\fi-?[0-9]+ ', r'\\fi0 ', line) + line = re.sub(r'\\ql', r'\\qr', line) + line = re.sub(r' *\[FLUSHRIGHT\] *', '', line) + special_after_space = after_space_multiplier * 1 + special_spacing = rtf_spacing['single'] + elif re.search(r'\[CENTER\]', line): + line = re.sub(r'\\fi-?[0-9]+ ', r'\\fi0 ', line) + line = re.sub(r'\\ql', r'\\qc', line) + line = re.sub(r' *\[CENTER\] *', '', line) + elif re.search(r'\[BOLDCENTER\]', line): + line = re.sub(r'\\fi-?[0-9]+ ', r'\\fi0 ', line) + line = re.sub(r'\\ql', r'\\qc \\b', line) + line = re.sub(r' *\[BOLDCENTER\] *', '', line) + elif indentation_command != '' and not re.search(r'\\widctlpar', line): + line = re.sub(r'\\fi-?[0-9]+ ', indentation_command, line) + if not re.search(r'\\s[0-9]', line): + if special_spacing: + spacing_command_to_use = special_spacing + else: + spacing_command_to_use = spacing_command + line = re.sub(r'\\pard ', r'\\pard ' + str(spacing_command_to_use) + str(border_text), line) + line = re.sub(r'\\pard\\intbl ', r'\\pard\\intbl ' + str(spacing_command_to_use) + str(border_text), line) + if not (re.search(r'\\fi0\\(endash|bullet)', line) or re.search(r'\\s[0-9]', line) or re.search(r'\\intbl', line)): + if special_after_space: + after_space_to_use = special_after_space + else: + after_space_to_use = after_space + if after_space_to_use > 0: + line = re.sub(r'\\sa[0-9]+ ', r'\\sa' + str(after_space_to_use) + ' ', line) + else: + line = re.sub(r'\\sa[0-9]+ ', r'\\sa0 ', line) + text += line + '\n' + text = re.sub(r'{\\pard \\sl[0-9]+\\slmult[0-9]+ \\ql \\f[0-9]+ \\sa[0-9]+ \\li[0-9]+ \\fi-?[0-9]*\s*\\par}', r'', text) + text = re.sub(r'\[MANUALSKIP\]', r'{\\pard \\sl0 \\ql \\f0 \\sa0 \\li0 \\fi0 \\par}', text) + return text + + +def metadata_filter(text, doc_format): + if doc_format == 'pdf': + text = re.sub(r'\*\*([^\*]+?)\*\*', r'\\begingroup\\bfseries \1\\endgroup {}', text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\*([^\*]+?)\*', r'\\begingroup\\itshape \1\\endgroup {}', text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\_\_([^\_]+?)\_\_', r'\\begingroup\\bfseries \1\\endgroup {}', text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\_([^\_]+?)\_*', r'\\begingroup\\itshape \1\\endgroup {}', text, flags=re.MULTILINE | re.DOTALL) + return text + + +def redact_latex(match): + return '\\redactword{' + str(escape_latex(match.group(1))) + '}' + + +def pdf_filter(text, metadata=None, question=None): + if metadata is None: + metadata = {} + text = text + "\n\n" + text = re.sub(r'\[\[([^\]]*)\]\]', r'\1', text) + text = re.sub(r'\[EMOJI ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_include_string(x, emoji=True, question=question), text) + text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', lambda x: image_include_string(x, question=question), text) + text = re.sub(r'\[FILE ([^,\]]+), *([0-9A-Za-z.%]+)\]', lambda x: image_include_string(x, question=question), text) + text = re.sub(r'\[FILE ([^,\]]+)\]', lambda x: image_include_string(x, question=question), text) + text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+), *([^\]]*)\]', qr_include_string, text) + text = re.sub(r'\[QR ([^,\]]+), *([0-9A-Za-z.%]+)\]', qr_include_string, text) + text = re.sub(r'\[QR ([^\]]+)\]', qr_include_string, text) + text = re.sub(r'\[MAP ([^\]]+)\]', '', text) + text = replace_fields(text) + # text = re.sub(r'\[FIELD ([^\]]+)\]', '', text) + text = re.sub(r'\[TARGET ([^\]]+)\]', '', text) + text = re.sub(r'\[YOUTUBE[^ ]* ([^\]]+)\]', '', text) + text = re.sub(r'\[VIMEO[^ ]* ([^\]]+)\]', '', text) + text = re.sub(r'\$\$+', '$', text) + text = re.sub(r'\\clearpage *\\clearpage', r'\\clearpage', text) + text = re.sub(r'\[BORDER\]\s*\[(BEGIN_TWOCOL|BEGIN_CAPTION|TIGHTSPACING|SINGLESPACING|DOUBLESPACING|START_INDENTATION|STOP_INDENTATION|NOINDENT|FLUSHLEFT|FLUSHRIGHT|CENTER|BOLDCENTER|INDENTBY[^\]]*)\]', r'[\1] [BORDER]', text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\[START_INDENTATION\]', r'\\setlength{\\parindent}{\\myindentamount}\\setlength{\\RaggedRightParindent}{\\parindent}', text) + text = re.sub(r'\[STOP_INDENTATION\]', r'\\setlength{\\parindent}{0in}\\setlength{\\RaggedRightParindent}{\\parindent}', text) + text = re.sub(r'\[BEGIN_CAPTION\](.+?)\[VERTICAL_LINE\]\s*(.+?)\[END_CAPTION\]', pdf_caption, text, flags=re.DOTALL) + text = re.sub(r'\[BEGIN_TWOCOL\](.+?)\[BREAK\]\s*(.+?)\[END_TWOCOL\]', pdf_two_col, text, flags=re.DOTALL) + text = re.sub(r'\[TIGHTSPACING\]\s*', r'\\singlespacing\\setlength{\\parskip}{0pt}\\setlength{\\parindent}{0pt}\\setlength{\\RaggedRightParindent}{\\parindent}', text) + text = re.sub(r'\[SINGLESPACING\]\s*', r'\\singlespacing\\setlength{\\parskip}{\\myfontsize}\\setlength{\\parindent}{0pt}\\setlength{\\RaggedRightParindent}{\\parindent}', text) + text = re.sub(r'\[DOUBLESPACING\]\s*', r'\\doublespacing\\setlength{\\parindent}{\\myindentamount}\\setlength{\\RaggedRightParindent}{\\parindent}', text) + text = re.sub(r'\[ONEANDAHALFSPACING\]\s*', r'\\onehalfspacing\\setlength{\\parindent}{\\myindentamount}\\setlength{\\RaggedRightParindent}{\\parindent}', text) + text = re.sub(r'\[TRIPLESPACING\]\s*', r'\\setlength{\\parindent}{\\myindentamount}\\setlength{\\RaggedRightParindent}{\\parindent}', text) + text = re.sub(r'\[NBSP\]', r'\\myshow{\\nonbreakingspace}', text) + text = re.sub(r'\[REDACTION_SPACE\]', r'\\redactword{~}\\hspace{0pt}', text) + text = re.sub(r'\[REDACTION_WORD ([^\]]+)\]', redact_latex, text) + text = re.sub(r'\[ENDASH\]', r'\\myshow{\\myendash}', text) + text = re.sub(r'\[EMDASH\]', r'\\myshow{\\myemdash}', text) + text = re.sub(r'\[HYPHEN\]', r'\\myshow{\\myhyphen}', text) + text = re.sub(r'\[CHECKBOX\]', r'{\\rule{0.3in}{0.4pt}}', text) + text = re.sub(r'\[BLANK\]', r'\\leavevmode{\\xrfill[-2pt]{0.4pt}}', text) + text = re.sub(r'\[BLANKFILL\]', r'\\leavevmode{\\xrfill[-2pt]{0.4pt}}', text) + text = re.sub(r'\[PAGEBREAK\]\s*', r'\\clearpage ', text) + text = re.sub(r'\[PAGENUM\]', r'\\myshow{\\thepage\\myxspace}', text) + text = re.sub(r'\[TOTALPAGES\]', r'\\myshow{\\pageref*{LastPage}\\myxspace}', text) + text = re.sub(r'\[SECTIONNUM\]', r'\\myshow{\\thesection\\myxspace}', text) + text = re.sub(r'\[VERTICALSPACE\] *', r'\\rule[-24pt]{0pt}{0pt}', text) + text = re.sub(r'\[NEWLINE\] *', r'\\newline ', text) + text = re.sub(r'\[NEWPAR\] *', r'\\par ', text) + text = re.sub(r'\[BR\] *', r'\\manuallinebreak ', text) + text = re.sub(r'\[TAB\] *', r'\\manualindent ', text) + text = re.sub(r' *\[END\] *', r'\n', text) + text = re.sub(r'\[NOINDENT\] *', r'\\noindent ', text) + text = re.sub(r'\[FLUSHLEFT\] *(.+?)\n *\n', flushleft_pdf, text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\[FLUSHRIGHT\] *(.+?)\n *\n', flushright_pdf, text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\[CENTER\] *(.+?)\n *\n', center_pdf, text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\[BOLDCENTER\] *(.+?)\n *\n', boldcenter_pdf, text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+)\] *(.+?)\n *\n', indentby_left_pdf, text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\[INDENTBY *([0-9\.]+ *[A-Za-z]+) *([0-9]+ *[A-Za-z]+)\] *(.+?)\n *\n', indentby_both_pdf, text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\[BORDER\] *(.+?)\n *\n', border_pdf, text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r'\s*\[SKIPLINE\]\s*', r'\\par\\myskipline ', text) + return text + + +def clean_markdown_to_latex(string): + string = re.sub(r'\s*\[SKIPLINE\]\s*', r'\\par\\myskipline ', string) + string = re.sub(r'^[\n ]+', '', string) + string = re.sub(r'[\n ]+$', '', string) + string = re.sub(r' *\n *$', '\n', string) + string = re.sub(r'\n{2,}', '[NEWLINE]', string) + string = re.sub(r'\[BR\]', '[NEWLINE]', string) + string = re.sub(r'\[(NOINDENT|FLUSHLEFT|FLUSHRIGHT|CENTER|BOLDCENTER|TIGHTSPACING|SINGLESPACING|DOUBLESPACING|START_INDENTATION|STOP_INDENTATION|PAGEBREAK)\]\s*', '', string) + string = re.sub(r'\*\*([^\*]+?)\*\*', r'\\textbf{\1}', string) + string = re.sub(r'\*([^\*]+?)\*', r'\\emph{\1}', string) + string = re.sub(r'(? 0 and file_info['width'] > 0: + scale = float(pixels)/float(file_info['width']) + # logmessage("scale is " + str(scale)) + if scale*float(file_info['height']) > float(MAX_HEIGHT_POINTS): + scale = float(MAX_HEIGHT_POINTS)/float(file_info['height']) + # logmessage("scale is " + str(scale)) + if scale*float(file_info['width']) > float(MAX_WIDTH_POINTS): + scale = float(MAX_WIDTH_POINTS)/float(file_info['width']) + # logmessage("scale is " + str(scale)) + # scale *= 100.0 + # logmessage("scale is " + str(scale)) + # scale = int(scale) + # logmessage("scale is " + str(scale)) + wtwips = int(scale*float(file_info['width'])*20.0) + htwips = int(scale*float(file_info['height'])*20.0) + image = Image(file_info['fullpath']) + image.Data = re.sub(r'\\picwgoal([0-9]+)', r'\\picwgoal' + str(wtwips), image.Data) + image.Data = re.sub(r'\\pichgoal([0-9]+)', r'\\pichgoal' + str(htwips), image.Data) + else: + image = Image(file_info['fullpath']) + if insert_page_breaks: + content = '\\page ' + else: + content = '' + # logmessage(content + image.Data) + return content + image.Data + + +def convert_percent(match): + percentage = match.group(1) + return str(float(percentage)/100.0) + '\\textwidth' + + +def image_include_string(match, emoji=False, question=None): + file_reference = match.group(1) + if question and file_reference in question.interview.images: + file_reference = question.interview.images[file_reference].get_reference() + try: + width = match.group(2) + assert width != 'None' + width = re.sub(r'^(.*)px', convert_pixels, width) + width = re.sub(r'^(.*)%', convert_percent, width) + if width == "full": + width = '\\textwidth' + except: + width = get_default_image_width() + if match.lastindex == 3: + alt_text = match.group(3) + else: + alt_text = None + file_info = file_finder(file_reference, question=question) + if 'path' in file_info and 'extension' in file_info: + convert_svg_to_eps(file_info) + if file_info['extension'] == 'gif': + with tempfile.NamedTemporaryFile(prefix="datemp", mode="wb", suffix=".png", delete=False) as png_file: + try: + with PIL.Image.open(file_info['fullpath']) as im: + im.save(png_file.name) + png_file.close() + file_info['path'] = png_file.name + file_info['fullpath'] = png_file.name + file_info['extension'] = 'png' + file_info['mimetype'] = 'image/png' + except BaseException as err: + logmessage("Could not convert GIF to PNG: " + err.__class__.__name__ + ": " + str(err)) + if 'mimetype' in file_info and file_info['mimetype']: + if re.search(r'^(audio|video)', file_info['mimetype']): + return '[reference to file type that cannot be displayed]' + if 'path' in file_info: + if 'extension' in file_info: + if file_info['extension'] in ['png', 'jpg', 'pdf', 'eps', 'jpe', 'jpeg', 'docx', 'rtf', 'doc', 'odt']: + if file_info['extension'] == 'pdf': + output = '\\includepdf[pages={-}]{' + file_info['path'] + '.pdf}' + elif file_info['extension'] in ('docx', 'rtf', 'doc', 'odt'): + if not os.path.isfile(file_info['path'] + '.pdf'): + fg_make_pdf_for_word_path(file_info['path'], file_info['extension']) + output = '\\includepdf[pages={-}]{' + file_info['path'] + '.pdf}' + else: + if alt_text: + alt_text_string = ', alt={' + re.sub(r'[{}]', '', alt_text) + '}' + else: + alt_text_string = '' + if emoji: + output = '\\raisebox{-.6\\dp\\strutbox}{\\mbox{\\includegraphics[width=' + width + alt_text_string + ']{' + file_info['path'] + '}}}' + else: + output = '\\mbox{\\includegraphics[width=' + width + alt_text_string + ']{' + file_info['path'] + '}}' + if width == '\\textwidth': + output = '\\clearpage ' + output + '\\clearpage ' + return output + return '[invalid graphics reference]' + + +def qr_include_string(match): + string = match.group(1) + try: + width = match.group(2) + assert width != 'None' + width = re.sub(r'^(.*)px', convert_pixels, width) + if width == "full": + width = '\\textwidth' + except: + width = get_default_image_width() + if match.lastindex == 3: + alt_text = match.group(3) + else: + alt_text = None + im = qrcode.make(string) + with tempfile.NamedTemporaryFile(prefix="datemp", suffix=".png", delete=False) as the_image: + # this_thread.temporary_resources.add(the_image.name) + im.save(the_image.name) + if alt_text: + alt_text_string = ', alt={' + re.sub(r'[{}]', '', alt_text) + '}' + else: + alt_text_string = '' + output = '\\mbox{\\includegraphics[width=' + width + alt_text_string + ']{' + the_image.name + '}}' + if width == '\\textwidth': + output = '\\clearpage ' + output + '\\clearpage ' + # logmessage("Output is " + output) + return output + + +def rtf_caption_table(match): + table_text = """\\trowd \\irow0\\irowband0\\lastrow \\ltrrow\\ts24\\trgaph108\\trleft0\\trbrdrt\\brdrs\\brdrw10 \\trbrdrl\\brdrs\\brdrw10 \\trbrdrb\\brdrs\\brdrw10 \\trbrdrr\\brdrs\\brdrw10 \\trbrdrh\\brdrs\\brdrw10 \\trbrdrv\\brdrs\\brdrw10 +\\trftsWidth1\\trftsWidthB3\\trftsWidthA3\\trautofit1\\trpaddl108\\trpaddr108\\trpaddfl3\\trpaddft3\\trpaddfb3\\trpaddfr3\\trcbpat1\\trcfpat1\\tblrsid1508006\\tbllkhdrrows\\tbllkhdrcols\\tbllknocolband\\tblind0\\tblindtype3 \\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrnone +\\clbrdrb\\brdrnone \\clbrdrr\\clshdng0\\brdrs\\brdrw10 \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx4680\\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrs\\brdrw10 \\clbrdrb\\brdrnone \\clbrdrr\\clshdng0\\brdrnone \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx9468\\pard\\plain \\ltrpar +\\ql \\li0\\ri0\\widctlpar\\intbl\\wrapdefault\\aspalpha\\aspnum\\faauto\\adjustright\\rin0\\lin0\\pararsid1508006\\yts24 \\rtlch\\fcs1 \\af0\\afs22\\alang1025 \\ltrch\\fcs0 \\fs22\\lang1033\\langfe1033\\cgrid\\langnp1033\\langfenp1033 { [SAVE][TIGHTSPACING][STOP_INDENTATION]""" + match.group(1) + """}{\\cell}{""" + match.group(2) + """[RESTORE]}{\\cell}\\pard\\plain \\ltrpar +\\ql \\li0\\ri0\\sa200\\sl276\\slmult1\\widctlpar\\intbl\\wrapdefault\\aspalpha\\aspnum\\faauto\\adjustright\\rin0\\lin0 \\rtlch\\fcs1 \\af0\\afs22\\alang1025 \\ltrch\\fcs0 \\fs24\\lang1033\\langfe1033\\cgrid\\langnp1033\\langfenp1033 {\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0 \\insrsid10753242 +\\trowd \\irow0\\irowband0\\lastrow \\ltrrow\\ts24\\trgaph108\\trleft0\\trbrdrt\\brdrs\\brdrw10 \\trbrdrl\\brdrs\\brdrw10 \\trbrdrb\\brdrs\\brdrw10 \\trbrdrr\\brdrs\\brdrw10 \\trbrdrh\\brdrs\\brdrw10 \\trbrdrv\\brdrs\\brdrw10 +\\trftsWidth1\\trftsWidthB3\\trftsWidthA3\\trautofit1\\trpaddl108\\trpaddr108\\trpaddfl3\\trpaddft3\\trpaddfb3\\trpaddfr3\\trcbpat1\\trcfpat1\\tblrsid1508006\\tbllkhdrrows\\tbllkhdrcols\\tbllknocolband\\tblind0\\tblindtype3 \\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrnone +\\clbrdrb\\brdrnone \\clbrdrr\\clshdng0\\brdrs\\brdrw10 \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx4680\\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrs\\brdrw10 \\clbrdrb\\brdrnone \\clbrdrr\\clshdng0\\brdrnone \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx9468\\row }""" + table_text += """\\pard \\ltrpar +\\qc \\li0\\ri0\\sb0\\sl240\\slmult1\\widctlpar\\wrapdefault\\aspalpha\\aspnum\\faauto\\adjustright\\rin0\\lin0\\itap0\\pararsid10753242""" + table_text = re.sub(r'\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0', r'\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0 \\sl240 \\slmult1', table_text) + return table_text + '[MANUALSKIP]' + + +def rtf_two_col(match): + table_text = """\\trowd \\irow0\\irowband0\\lastrow \\ltrrow\\ts24\\trgaph108\\trleft0\\trbrdrt\\brdrs\\brdrw10 \\trbrdrl\\brdrs\\brdrw10 \\trbrdrb\\brdrs\\brdrw10 \\trbrdrr\\brdrs\\brdrw10 \\trbrdrh\\brdrs\\brdrw10 \\trbrdrv\\brdrs\\brdrw10 +\\trftsWidth1\\trftsWidthB3\\trftsWidthA3\\trautofit1\\trpaddl108\\trpaddr108\\trpaddfl3\\trpaddft3\\trpaddfb3\\trpaddfr3\\trcbpat1\\trcfpat1\\tblrsid1508006\\tbllkhdrrows\\tbllkhdrcols\\tbllknocolband\\tblind0\\tblindtype3 \\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrnone +\\clbrdrb\\brdrnone \\clbrdrr\\brdrnone \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx4680\\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrnone \\clbrdrb\\brdrnone \\clbrdrr\\clshdng0\\brdrnone \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx9468\\pard\\plain \\ltrpar +\\ql \\li0\\ri0\\widctlpar\\intbl\\wrapdefault\\aspalpha\\aspnum\\faauto\\adjustright\\rin0\\lin0\\pararsid1508006\\yts24 \\rtlch\\fcs1 \\af0\\afs22\\alang1025 \\ltrch\\fcs0 \\fs22\\lang1033\\langfe1033\\cgrid\\langnp1033\\langfenp1033 {\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0 \\insrsid2427490 [SAVE][TIGHTSPACING][STOP_INDENTATION]""" + match.group(1) + """}{\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0 \\insrsid10753242\\charrsid2427490 \\cell}{""" + match.group(2) + """[RESTORE]}{\\cell}\\pard\\plain \\ltrpar +\\ql \\li0\\ri0\\sa200\\sl276\\slmult1\\widctlpar\\intbl\\wrapdefault\\aspalpha\\aspnum\\faauto\\adjustright\\rin0\\lin0 \\rtlch\\fcs1 \\af0\\afs22\\alang1025 \\ltrch\\fcs0 \\fs24\\lang1033\\langfe1033\\cgrid\\langnp1033\\langfenp1033 {\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0 \\insrsid10753242 +\\trowd \\irow0\\irowband0\\lastrow \\ltrrow\\ts24\\trgaph108\\trleft0\\trbrdrt\\brdrs\\brdrw10 \\trbrdrl\\brdrs\\brdrw10 \\trbrdrb\\brdrs\\brdrw10 \\trbrdrr\\brdrs\\brdrw10 \\trbrdrh\\brdrs\\brdrw10 \\trbrdrv\\brdrs\\brdrw10 +\\trftsWidth1\\trftsWidthB3\\trftsWidthA3\\trautofit1\\trpaddl108\\trpaddr108\\trpaddfl3\\trpaddft3\\trpaddfb3\\trpaddfr3\\trcbpat1\\trcfpat1\\tblrsid1508006\\tbllkhdrrows\\tbllkhdrcols\\tbllknocolband\\tblind0\\tblindtype3 \\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrnone +\\clbrdrb\\brdrnone \\clbrdrr\\brdrnone \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx4680\\clvertalc\\clbrdrt\\brdrnone \\clbrdrl\\brdrnone \\clbrdrb\\brdrnone \\clbrdrr\\clshdng0\\brdrnone \\cltxlrtb\\clftsWidth3\\clwWidth4732 \\cellx9468\\row }""" + table_text += """\\pard \\ltrpar +\\qc \\li0\\ri0\\sb0\\sl240\\slmult1\\widctlpar\\wrapdefault\\aspalpha\\aspnum\\faauto\\adjustright\\rin0\\lin0\\itap0\\pararsid10753242""" + table_text = re.sub(r'\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0', r'\\rtlch\\fcs1 \\af0 \\ltrch\\fcs0 \\sl240 \\slmult1', table_text) + return table_text + '[MANUALSKIP]' diff --git a/docassemble_base/docassemble/base/filter/utils.py b/docassemble_base/docassemble/base/filter/utils.py new file mode 100644 index 000000000..501b67e5e --- /dev/null +++ b/docassemble_base/docassemble/base/filter/utils.py @@ -0,0 +1,131 @@ +import re +import tempfile +import PIL +from cairosvg import svg2png, svg2eps +from ..logger import logmessage + +zerowidth = '\u200B' # pylint: disable=invalid-name + +list_types = ['1', 'A', 'a', 'I', 'i'] + +DEFAULT_IMAGE_WIDTH = '4in' + + +def set_default_image_width(width): + global DEFAULT_IMAGE_WIDTH + DEFAULT_IMAGE_WIDTH = str(width) + + +def get_default_image_width(): + return DEFAULT_IMAGE_WIDTH + +unit_multipliers = {'twips': 0.0500, 'hp': 0.5, 'in': 72, 'pt': 1, 'px': 1, 'em': 12, 'cm': 28.346472} + + +def pixels_in(length): + m = re.search(r"([0-9.]+) *([a-z]+)", str(length).lower()) + if m: + value = float(m.group(1)) + unit = m.group(2) + # logmessage("value is " + str(value) + " and unit is " + unit) + if unit in unit_multipliers: + size = float(unit_multipliers[unit]) * value + # logmessage("size is " + str(size)) + return int(size) + logmessage("Could not read " + str(length)) + return 300 + + +def convert_length(length, unit): + value = pixels_in(length) + if unit in unit_multipliers: + size = float(value)/float(unit_multipliers[unit]) + return int(size) + logmessage("Unit " + str(unit) + " is not a valid unit") + return 300 + + +def replace_fields(string, status=None, embedder=None): + if not re.search(r'\[FIELD ', string): + return string + matches = [] + in_match = False + start_match = None + depth = 0 + i = 0 + while i < len(string): + if string[i:i+7] == '[FIELD ': + in_match = True + start_match = i + i += 7 + continue + if in_match: + if string[i] == '[': + depth += 1 + elif string[i] == ']': + if depth == 0: + i += 1 + matches.append((start_match, i)) + in_match = False + continue + depth -= 1 + i += 1 + + field_strings = [] + for (start, end) in matches: + field_strings.append(string[start:end]) + # logmessage(repr(field_strings)) + for field_string in field_strings: + if embedder is None: + string = string.replace(field_string, 'ERROR: FIELD cannot be used here') + else: + string = string.replace(field_string, embedder(status, field_string)) + return string + + +def repeat_along(chars, match): + output = chars * len(match.group(1)) + # logmessage("Output is " + repr(output)) + return output + + +def convert_pixels(match): + pixels = match.group(1) + return str(int(pixels)/72.0) + "in" + + +def convert_svg_to_eps(file_info): + try: + if file_info['extension'] == 'svg': + with tempfile.NamedTemporaryFile(prefix="datemp", mode="wb", suffix=".eps", delete=False) as eps_file: + with open(file_info['fullpath'], 'rb') as fp: + svg2eps(file_obj=fp, write_to=eps_file) + file_info['path'] = eps_file.name + file_info['fullpath'] = eps_file.name + file_info['extension'] = 'eps' + file_info['mimetype'] = 'application/postscript' + eps_file.close() + except BaseException as err: + logmessage("Failure to convert SVG to EPS: " + err.__class__.__name__ + ": " + str(err)) + + +def convert_svg_to_png(file_info): + try: + if file_info['extension'] == 'svg': + with tempfile.NamedTemporaryFile(prefix="datemp", mode="wb", suffix=".png", delete=False) as png_file: + with open(file_info['fullpath'], 'rb') as fp: + svg2png(file_obj=fp, write_to=png_file, dpi=300) + png_file.flush() + with PIL.Image.open(png_file.name) as im: + file_info['width'], file_info['height'] = im.size + file_info['path'] = png_file.name + file_info['fullpath'] = png_file.name + file_info['extension'] = 'png' + file_info['mimetype'] = 'image/png' + png_file.close() + except BaseException as err: + logmessage("Failure to convert SVG to PNG: " + err.__class__.__name__ + ": " + str(err)) + + +def sanitize_xml(text): + return re.sub(r'{([{%#])', '{' + zerowidth + r'\1', re.sub(r'([}%#])}', r'\1' + zerowidth + '}', text)) diff --git a/docassemble_base/docassemble/base/functions.py b/docassemble_base/docassemble/base/functions.py index 841b7486c..b9007472e 100644 --- a/docassemble_base/docassemble/base/functions.py +++ b/docassemble_base/docassemble/base/functions.py @@ -1,5 +1,11 @@ +# ruff: noqa: F401 +# pylint: disable=unused-import +# mypy: disable-error-code="var-annotated" import re +import sys import types +import traceback +from types import SimpleNamespace import os import locale import decimal @@ -10,43 +16,172 @@ import json import ast import datetime -import threading import random from collections.abc import Iterable from unicodedata import normalize from enum import Enum from pathlib import Path import importlib.resources -import sys import astunparse -import tzlocal import us import pycountry -import markdown import ruamel.yaml -from types import SimpleNamespace -from docassemble.base.save_status import SS_NEW, SS_OVERWRITE, SS_IGNORE -from docassemble.base.pattern import pattern_en, pattern_es, pattern_de, pattern_fr, pattern_it, pattern_nl - from pylatex.utils import escape_latex # import operator -import titlecase from user_agents import parse as ua_parse import phonenumbers import werkzeug.utils -import num2words -from jinja2.runtime import Undefined -from docassemble.base.logger import logmessage # pylint: disable=ungrouped-imports -from docassemble.base.error import ForcedNameError, QuestionError, ResponseError, CommandError, BackgroundResponseError, BackgroundResponseActionError, ForcedReRun, DAError, DANameError, DAInvalidFilename +from docassemble.base.background import bg_action +from docassemble.base.error import ( + ForcedNameError, + QuestionError, + ResponseError, + CommandError, + BackgroundResponseError, + BackgroundResponseActionError, + ForcedReRun, + DAError, + DANameError, + DAInvalidFilename, +) from docassemble.base.generate_key import random_string +from docassemble.base.hooks import ( + absolute_filename, + add_privilege, + chat_partners_available as server_chat_partners_available, + delete_record, + get_button_class_prefix, + get_chat_log as server_get_chat_log, + get_configuration, + get_debug_status, + get_default_timezone, + get_hostname, + get_login_url, + get_permissions_of_privilege, + get_privileges_list, + get_referer, + get_server_redis, + get_short_code, + get_url, + navigation_bar, + read_records, + release_lock, + remove_privilege, + retrieve_emails, + server_create_session, + server_create_user, + server_get_question_data, + server_get_secret, + server_get_session_variables, + server_get_user_info, + server_get_user_list, + server_go_back_in_session, + server_interview_menu, + server_invite_user, + server_run_action_in_session, + server_set_session_variables, + server_set_user_info, + transform_json_variables, + url_finder, + url_for, + user_interviews, + write_answer_json, + write_record, +) +from docassemble.base.language.capitalization import capitalize # noqa: F401 # pylint: disable=unused-import +from docassemble.base.language.control import ( + get_language, + set_language, + set_country, + get_country, + get_dialect, + get_voice, + set_locale, + get_locale, + update_locale, +) +from docassemble.base.language.core import ( + language_functions, + ensure_definition, + language_function_constructor, + update_language_function, +) +from docassemble.base.language.currency import ( + currency, + currency_symbol, + get_currency_symbol, +) +from docassemble.base.language.language import ( + comma_list, + comma_and_list, + quantity_noun, + verb_past, + verb_present, + noun_plural, + noun_singular, + indefinite_article, + period_list, + name_suffix, + title_case, + add_separators, + its, + the, + her, + does_a_b, + was_a_b, + do_you, + this, + their, + possessify, + did_you, + salutation, + his, + a_in_the_b, + some, + these, + have_you, + your, + did_a_b, + possessify_long, + were_you, + is_word, + has_a_b, +) +from docassemble.base.language.numbers import ( + number_to_word, + ordinal, + nice_number, + ordinal_functions, + ordinal_number, + update_nice_numbers, + update_ordinal_numbers, + update_ordinal_function, + string_to_number, +) +from docassemble.base.language.utils import fix_punctuation +from docassemble.base.language.words import ( + words, + word, + update_word_collection, + word_collection, +) +from docassemble.base.logger import logmessage +from docassemble.base.save_status import SS_NEW, SS_OVERWRITE, SS_IGNORE +from docassemble.base.thread_context import ( + this_thread, + get_current_user_dict, + get_old_user_dict, +) import docassemble.base.astparser + +ordinal_function = ordinal FileType = IOBase equals_byte = bytes('=', 'utf-8') TypeType = type(type(None)) locale.setlocale(locale.LC_ALL, '') contains_volatile = re.compile(r'^(x\.|x\[|.*\[[ijklmn]\])') match_brackets_or_dot = re.compile(r'(\[.+?\]|\.[a-zA-Z_][a-zA-Z0-9_]*)') -python313 = sys.version_info >= (3, 13) +# python313 = sys.version_info >= (3, 13) __all__ = ['alpha', 'roman', 'item_label', 'ordinal', 'ordinal_number', 'comma_list', 'word', 'get_language', 'set_language', 'get_dialect', 'set_country', 'get_country', 'get_locale', 'set_locale', 'comma_and_list', 'need', 'nice_number', 'quantity_noun', 'currency_symbol', 'verb_past', 'verb_present', 'noun_plural', 'noun_singular', 'indefinite_article', 'capitalize', 'space_to_underscore', 'force_ask', 'period_list', 'name_suffix', 'currency', 'static_image', 'title_case', 'url_of', 'process_action', 'url_action', 'get_info', 'set_info', 'get_config', 'prevent_going_back', 'qr_code', 'action_menu_item', 'from_b64_json', 'defined', 'value', 'message', 'response', 'json_response', 'command', 'background_response', 'background_response_action', 'single_paragraph', 'quote_paragraphs', 'location_returned', 'location_known', 'user_lat_lon', 'interview_url', 'interview_url_action', 'interview_url_as_qr', 'interview_url_action_as_qr', 'interview_email', 'get_emails', 'action_arguments', 'action_argument', 'get_default_timezone', 'user_logged_in', 'user_privileges', 'user_has_privilege', 'user_info', 'current_context', 'background_action', 'background_response', 'background_response_action', 'us', 'set_live_help_status', 'chat_partners_available', 'phone_number_in_e164', 'phone_number_formatted', 'phone_number_is_valid', 'countries_list', 'country_name', 'write_record', 'read_records', 'delete_record', 'variables_as_json', 'all_variables', 'language_from_browser', 'device', 'plain', 'bold', 'italic', 'subdivision_type', 'indent', 'raw', 'fix_punctuation', 'set_progress', 'get_progress', 'referring_url', 'undefine', 'invalidate', 'dispatch', 'yesno', 'noyes', 'phone_number_part', 'log', 'encode_name', 'decode_name', 'interview_list', 'interview_menu', 'server_capabilities', 'session_tags', 'get_chat_log', 'get_user_list', 'get_user_info', 'set_user_info', 'get_user_secret', 'create_user', 'invite_user', 'create_session', 'get_session_variables', 'set_session_variables', 'go_back_in_session', 'manage_privileges', 'redact', 'forget_result_of', 're_run_logic', 'reconsider', 'get_question_data', 'set_save_status', 'single_to_double_newlines', 'verbatim', 'add_separators', 'store_variables_snapshot', 'update_terms', 'set_variables', 'language_name', 'run_action_in_session'] @@ -179,22 +314,22 @@ def wrap_up(): file_object.commit() -def set_gathering_mode(mode, instanceName): - # logmessage("set_gathering_mode: " + str(instanceName) + " with mode " + str(mode)) +def set_gathering_mode(mode, instance_name): + # logmessage("set_gathering_mode: " + str(instance_name) + " with mode " + str(mode)) if mode: - if instanceName not in this_thread.gathering_mode: + if instance_name not in this_thread.gathering_mode: # logmessage("set_gathering_mode: using " + str(get_current_variable())) - this_thread.gathering_mode[instanceName] = get_current_variable() + this_thread.gathering_mode[instance_name] = get_current_variable() else: try: - del this_thread.gathering_mode[instanceName] + del this_thread.gathering_mode[instance_name] except KeyError: pass -def get_gathering_mode(instanceName): - # logmessage("get_gathering_mode: " + str(instanceName)) - if instanceName not in this_thread.gathering_mode: +def get_gathering_mode(instance_name): + # logmessage("get_gathering_mode: " + str(instance_name)) + if instance_name not in this_thread.gathering_mode: # logmessage("get_gathering_mode: returning False") return False # logmessage("get_gathering_mode: returning True") @@ -208,9 +343,9 @@ def reset_gathering_mode(*pargs): return var = pargs[0] todel = [] - for instanceName, curVar in this_thread.gathering_mode.items(): - if curVar == var: - todel.append(instanceName) + for instance_name, current_var in this_thread.gathering_mode.items(): + if current_var == var: + todel.append(instance_name) # logmessage("reset_gathering_mode: deleting " + repr([y for y in todel])) for item in todel: try: @@ -246,7 +381,7 @@ def get_chat_log(utc=False, timezone=None): Returns: list: A list of chat messages for the current interview session. """ - return server.get_chat_log(this_thread.current_info.get('yaml_filename', None), this_thread.current_info.get('session', None), this_thread.current_info.get('secret', None), utc=utc, timezone=timezone) + return server_get_chat_log(this_thread.current_info.get('yaml_filename', None), this_thread.current_info.get('session', None), this_thread.current_info.get('secret', None), utc=utc, timezone=timezone) def get_current_package(): @@ -847,7 +982,7 @@ def user_has_privilege(*pargs): return False -class AttachmentInfo: +class AttachmentInfo(SimpleNamespace): pass @@ -899,7 +1034,7 @@ def current_filename(self): @property def current_section(self): try: - return this_thread.current_section or get_user_dict()['nav'].current + return this_thread.current_section or get_current_user_dict()['nav'].current except: return None @@ -913,7 +1048,7 @@ def inside_of(self): @property def request_url(self): try: - info = server.get_url() + info = get_url() except: info = {} return info @@ -1049,7 +1184,7 @@ def privileges(self): def permissions(self): enabled_privileges = set() for privilege in user_privileges(): - enabled_privileges.update(server.get_permissions_of_privilege(privilege, privileged=True)) + enabled_privileges.update(get_permissions_of_privilege(privilege, privileged=True)) return list(enabled_privileges) @property @@ -1106,7 +1241,7 @@ def current_filename(self): def current_section(self): warn_if_not_warned('user_info', 'current_section', 'current_context') try: - return this_thread.current_section or get_user_dict()['nav'].current + return this_thread.current_section or get_current_user_dict()['nav'].current except: return None @@ -1272,7 +1407,7 @@ def chat_partners_available(*pargs, **kwargs): if the_user_id == 'tNone': logmessage("chat_partners_available: unable to get temporary user id") return {'peer': 0, 'help': 0} - return server.chat_partners_available(session_id, yaml_filename, the_user_id, mode, partner_roles) + return server_chat_partners_available(session_id, yaml_filename, the_user_id, mode, partner_roles) def interview_email(key=None, index=None): @@ -1293,8 +1428,8 @@ def interview_email(key=None, index=None): """ if key is None and index is not None: raise DAError("interview_email: if you provide an index you must provide a key") - domain = server.daconfig.get('incoming mail domain', server.daconfig.get('external hostname', server.hostname)) - return server.get_short_code(key=key, index=index) + '@' + domain + domain = get_configuration().get('incoming mail domain', get_configuration().get('external hostname', get_hostname())) + return get_short_code({"key": key, "index": index}) + '@' + domain def get_emails(key=None, index=None): @@ -1313,7 +1448,7 @@ def get_emails(key=None, index=None): list: A list of objects representing e-mail addresses and their received messages. """ - return server.retrieve_emails(key=key, index=index) + return retrieve_emails(key=key, index=index) def modify_i_argument(args): @@ -1384,7 +1519,7 @@ def interview_url(**kwargs): is_new = False url = None if the_style == 'short': - for k, v in server.daconfig.get('dispatch').items(): + for k, v in get_configuration().get('dispatch').items(): if v == args['i']: args['dispatch'] = k del args['i'] @@ -1425,12 +1560,13 @@ def interview_url(**kwargs): def temp_redirect(url, expire_seconds, do_local, one_time): + redis_server = get_server_redis() while True: code = random_string(32) the_key = 'da:temporary_url:' + code - if server.server_redis.get(the_key) is None: + if redis_server.get(the_key) is None: break - pipe = server.server_redis.pipeline() + pipe = redis_server.pipeline() if one_time: pipe.set(the_key, json.dumps({'url': url, 'once': True})) else: @@ -1438,8 +1574,8 @@ def temp_redirect(url, expire_seconds, do_local, one_time): pipe.expire(the_key, expire_seconds) pipe.execute() if do_local: - return server.url_for('run_temp', c=code) - return server.url_for('run_temp', c=code, _external=True) + return url_for('main.run_temp', c=code) + return url_for('main.run_temp', c=code, _external=True) def set_parts(**kwargs): @@ -1680,7 +1816,7 @@ def interview_url_action(action, **kwargs): pass url = None if the_style == 'short': - for k, v in server.daconfig.get('dispatch').items(): + for k, v in get_configuration().get('dispatch').items(): if v == args['i']: args['dispatch'] = k del args['i'] @@ -1886,7 +2022,7 @@ def set_save_status(status): this_thread.misc['save_status'] = SS_OVERWRITE if status == 'ignore': this_thread.misc['save_status'] = SS_IGNORE - server.release_lock(this_thread.current_info['session'], this_thread.current_info['yaml_filename']) + release_lock(this_thread.current_info['session'], this_thread.current_info['yaml_filename']) class DANav: @@ -2055,194 +2191,144 @@ def show_sections(self, style='inline', show_links=None): if style == "inline": the_class = 'danavlinks dainline' interior_class = 'dainlineinside' - a_class = "btn " + server.button_class_prefix + "secondary danavlink " + a_class = "btn " + get_button_class_prefix() + "secondary danavlink " else: if not self.visible(): return '' the_class = 'danavlinks' interior_class = None a_class = None - return '
' + "\n" + server.navigation_bar(self, this_thread.interview, wrapper=False, inner_div_class=interior_class, a_class=a_class, show_links=show_links, show_nesting=False, include_arrows=True) + '
' + "\n" + return '
' + "\n" + navigation_bar(self, this_thread.interview, wrapper=False, inner_div_class=interior_class, a_class=a_class, show_links=show_links, show_nesting=False, include_arrows=True) + '
' + "\n" # word('This field is required.') # word('Country Code') # word('First Subdivision') # word('Second Subdivision') # word('Third Subdivision') -word_collection = { - 'en': { - 'This field is required.': 'You need to fill this in.', - "Country Code": 'Country Code (e.g., "us")', - "First Subdivision": 'State Abbreviation (e.g., "NY")', - "Second Subdivision": "County", - "Third Subdivision": "Municipality", - } -} - -ordinal_numbers = { -} - -nice_numbers = { -} - - -class WebFunc: - pass -server = WebFunc() - - -def null_func(*pargs, **kwargs): # pylint: disable=unused-argument - return None -def null_func_dict(*pargs, **kwargs): # pylint: disable=unused-argument - return {} - - -def null_func_str(*pargs, **kwargs): # pylint: disable=unused-argument - return '' - - -def null_func_obj(*pargs, **kwargs): # pylint: disable=unused-argument - return WebFunc() - - -def null_func_func(*pargs, **kwargs): # pylint: disable=unused-argument - return null_func - -server.SavedFile = null_func_obj -server.absolute_filename = null_func -server.add_privilege = null_func -server.add_user_privilege = null_func -server.alchemy_url = null_func_str -server.connect_args = null_func_str -server.applock = null_func -server.bg_action = null_func -server.ocr_google_in_background = null_func -server.button_class_prefix = 'btn-' -server.chat_partners_available = null_func -server.chord = null_func_func -server.create_user = null_func -server.invite_user = null_func -server.daconfig = {} -server.debug = False -server.debug_status = False -server.default_country = 'US' -server.default_dialect = 'us' -server.default_voice = None -server.default_language = 'en' -server.default_locale = 'US.utf8' -try: - server.default_timezone = tzlocal.get_localzone_name() -except: - server.default_timezone = 'America/New_York' -server.delete_answer_json = null_func -server.delete_record = null_func -server.fg_make_pdf_for_word_path = null_func -server.fg_make_png_for_pdf = null_func -server.fg_make_png_for_pdf_path = null_func -server.file_finder = null_func_dict -server.file_number_finder = null_func_dict -server.file_privilege_access = null_func -server.file_set_attributes = null_func -server.file_user_access = null_func -server.fix_pickle_obj = null_func_dict -server.generate_csrf = null_func -server.get_chat_log = null_func -server.get_ext_and_mimetype = null_func -server.get_new_file_number = null_func -server.get_privileges_list = null_func -server.get_question_data = null_func -server.get_secret = null_func -server.get_session_variables = null_func -server.get_short_code = null_func -server.get_sms_session = null_func_dict -server.get_user_info = null_func -server.get_user_list = null_func -server.get_user_object = null_func_obj -server.go_back_in_session = null_func -server.hostname = 'localhost' -server.initiate_sms_session = null_func -server.interview_menu = null_func -server.main_page_parts = {} -server.make_png_for_pdf = null_func -server.make_user_inactive = null_func -server.navigation_bar = null_func -server.ocr_finalize = null_func -server.ocr_page = null_func -server.path_from_reference = null_func_str -server.read_answer_json = null_func -server.read_records = null_func -server.remove_privilege = null_func -server.remove_user_privilege = null_func -server.retrieve_emails = null_func -server.save_numbered_file = null_func -server.send_fax = null_func -server.send_mail = null_func -server.server_redis = None -server.server_redis_user = None -server.server_sql_defined = null_func -server.server_sql_delete = null_func -server.server_sql_get = null_func -server.server_sql_keys = null_func -server.server_sql_set = null_func -server.create_session = null_func -server.set_session_variables = null_func -server.set_user_info = null_func -server.sms_body = null_func_dict -server.task_ready = null_func -server.terminate_sms_session = null_func -server.twilio_config = {} -server.url_finder = null_func_dict -server.url_for = null_func -server.user_id_dict = null_func -server.user_interviews = null_func -server.variables_snapshot_connection = null_func -server.wait_for_task = null_func -server.worker_convert = null_func -server.write_answer_json = null_func -server.write_record = null_func -server.to_text = null_func_str -server.transform_json_variables = null_func -server.get_login_url = null_func_dict -server.run_action_in_session = null_func_dict -server.invite_user = null_func -server.get_url = null_func_dict - - -def write_record(key, data): - """Store data in the SQL database under the given key. - - Args: - key (str): A string key to associate with the record. - data: The data to store. Must be pickleable. +# class WebFunc: +# pass +# server = WebFunc() - Returns: - int: The unique integer ID of the saved record. - """ - return server.write_record(key, data) - - -def read_records(key): - """Return all records stored under the given key. - Args: - key (str): The string key used when calling ``write_record()``. +# def null_func(*pargs, **kwargs): # pylint: disable=unused-argument +# return None - Returns: - dict: A dictionary mapping unique integer record IDs to the stored data. - """ - return server.read_records(key) +# def null_func_dict(*pargs, **kwargs): # pylint: disable=unused-argument +# return {} -def delete_record(key, the_id): - """Delete a record from the SQL database by key and ID. - Args: - key (str): The string key associated with the record. - the_id (int): The unique integer ID of the record to delete. - """ - return server.delete_record(key, the_id) +# def null_func_str(*pargs, **kwargs): # pylint: disable=unused-argument +# return '' + + +# def null_func_obj(*pargs, **kwargs): # pylint: disable=unused-argument +# return WebFunc() + + +# def null_func_func(*pargs, **kwargs): # pylint: disable=unused-argument +# return null_func + +# server.SavedFile = null_func_obj +# server.absolute_filename = null_func +# server.add_privilege = null_func +# server.add_user_privilege = null_func +# server.alchemy_url = null_func_str +# server.connect_args = null_func_str +# server.applock = null_func +# server.bg_action = null_func +# server.ocr_google_in_background = null_func +# server.button_class_prefix = 'btn-' +# server.chat_partners_available = null_func +# server.chord = null_func_func +# server.create_user = null_func +# server.invite_user = null_func +# server.daconfig = {} +# server.debug = False +# server.debug_status = False +# server.default_country = 'US' +# server.default_dialect = 'us' +# server.default_voice = None +# server.default_language = 'en' +# server.default_locale = 'US.utf8' +# try: +# server.default_timezone = tzlocal.get_localzone_name() +# except: +# server.default_timezone = 'America/New_York' +# server.delete_answer_json = null_func +# server.delete_record = null_func +# server.fg_make_pdf_for_word_path = null_func +# server.fg_make_png_for_pdf = null_func +# server.fg_make_png_for_pdf_path = null_func +# server.file_finder = null_func_dict +# server.file_number_finder = null_func_dict +# server.file_privilege_access = null_func +# server.file_set_attributes = null_func +# server.file_user_access = null_func +# server.fix_pickle_obj = null_func_dict +# server.generate_csrf = null_func +# server.get_chat_log = null_func +# server.get_ext_and_mimetype = null_func +# server.get_new_file_number = null_func +# server.get_privileges_list = null_func +# server.get_question_data = null_func +# server.get_secret = null_func +# server.get_session_variables = null_func +# server.get_short_code = null_func +# server.get_sms_session = null_func_dict +# server.get_user_info = null_func +# server.get_user_list = null_func +# server.get_user_object = null_func_obj +# server.go_back_in_session = null_func +# server.hostname = 'localhost' +# server.initiate_sms_session = null_func +# server.interview_menu = null_func +# server.main_page_parts = {} +# server.make_png_for_pdf = null_func +# server.make_user_inactive = null_func +# server.navigation_bar = null_func +# server.ocr_finalize = null_func +# server.ocr_page = null_func +# server.path_from_reference = null_func_str +# server.read_answer_json = null_func +# server.read_records = null_func +# server.remove_privilege = null_func +# server.remove_user_privilege = null_func +# server.retrieve_emails = null_func +# server.save_numbered_file = null_func +# server.send_fax = null_func +# server.send_mail = null_func +# server.server_redis = None +# server.server_redis_user = None +# server.server_sql_defined = null_func +# server.server_sql_delete = null_func +# server.server_sql_get = null_func +# server.server_sql_keys = null_func +# server.server_sql_set = null_func +# server.create_session = null_func +# server.set_session_variables = null_func +# server.set_user_info = null_func +# server.sms_body = null_func_dict +# server.task_ready = null_func +# server.terminate_sms_session = null_func +# server.twilio_config = {} +# server.url_finder = null_func_dict +# server.url_for = null_func +# server.user_id_dict = null_func +# server.user_interviews = null_func +# server.variables_snapshot_connection = null_func +# server.wait_for_task = null_func +# server.worker_convert = null_func +# server.write_answer_json = null_func +# server.write_record = null_func +# server.to_text = null_func_str +# server.transform_json_variables = null_func +# server.get_login_url = null_func_dict +# server.run_action_in_session = null_func_dict +# server.invite_user = null_func +# server.get_url = null_func_dict def url_of(file_reference, **kwargs): @@ -2289,7 +2375,7 @@ def url_of(file_reference, **kwargs): for param in ('expire', 'url_args', 'next', 'i', 'session', 'resume_existing'): if param in kwargs and kwargs[param] is not None: info[param] = kwargs[param] - result = server.get_login_url(**info) + result = get_login_url(**info) if result['status'] == 'success': return result['url'] raise DAError("url_of: " + result['message']) @@ -2299,7 +2385,7 @@ def url_of(file_reference, **kwargs): kwargs['_question'] = get_current_question() if kwargs.get('attachment', False): kwargs['_attachment'] = True - return server.url_finder(file_reference, **kwargs) + return url_finder(file_reference, **kwargs) def server_capabilities(): @@ -2315,20 +2401,20 @@ def server_capabilities(): dict: A dictionary mapping capability names to True/False values. """ result = {'sms': False, 'fax': False, 'google_login': False, 'facebook_login': False, 'auth0_login': False, 'keycloak_login': False, 'authentik_login': False, 'azure_login': False, 'miniorange_login': False, 'phone_login': False, 'voicerss': False, 's3': False, 'azure': False, 'github': False, 'pypi': False, 'googledrive': False, 'google_maps': False} - if 'twilio' in server.daconfig and isinstance(server.daconfig['twilio'], (list, dict)): - if isinstance(server.daconfig['twilio'], list): - tconfigs = server.daconfig['twilio'] + if 'twilio' in get_configuration() and isinstance(get_configuration()['twilio'], (list, dict)): + if isinstance(get_configuration()['twilio'], list): + tconfigs = get_configuration()['twilio'] else: - tconfigs = [server.daconfig['twilio']] + tconfigs = [get_configuration()['twilio']] for tconfig in tconfigs: if 'enable' in tconfig and not tconfig['enable']: continue result['sms'] = True if tconfig.get('fax', False): result['fax'] = True - if 'phone login' in server.daconfig: + if 'phone login' in get_configuration(): result['phone_login'] = True - if 'oauth' in server.daconfig and isinstance(server.daconfig['oauth'], dict): + if 'oauth' in get_configuration() and isinstance(get_configuration()['oauth'], dict): oauth_providers = [ ('google', 'google_login'), ('facebook', 'facebook_login'), @@ -2341,35 +2427,35 @@ def server_capabilities(): ('github', 'github') ] for provider, result_key in oauth_providers: - if provider in server.daconfig['oauth'] and isinstance(server.daconfig['oauth'][provider], dict) and ('enable' not in server.daconfig['oauth'][provider] or server.daconfig['oauth'][provider]['enable']): + if provider in get_configuration()['oauth'] and isinstance(get_configuration()['oauth'][provider], dict) and ('enable' not in get_configuration()['oauth'][provider] or get_configuration()['oauth'][provider]['enable']): result[result_key] = True - if 'pypi' in server.daconfig and server.daconfig['pypi'] is True: + if 'pypi' in get_configuration() and get_configuration()['pypi'] is True: result['pypi'] = True - if 'google' in server.daconfig and isinstance(server.daconfig['google'], dict) and ('google maps api key' in server.daconfig['google'] or 'api key' in server.daconfig['google']): + if 'google' in get_configuration() and isinstance(get_configuration()['google'], dict) and ('google maps api key' in get_configuration()['google'] or 'api key' in get_configuration()['google']): result['google_maps'] = True for key in ['voicerss', 's3', 'azure']: - if key in server.daconfig and isinstance(server.daconfig[key], dict): - if not ('enable' in server.daconfig[key] and not server.daconfig[key]['enable']): + if key in get_configuration() and isinstance(get_configuration()[key], dict): + if not ('enable' in get_configuration()[key] and not get_configuration()[key]['enable']): result[key] = True return result # def generate_csrf(*pargs, **kwargs): -# return server.generate_csrf(*pargs, **kwargs) +# return generate_csrf(*pargs, **kwargs) # def chat_partners(*pargs, **kwargs): # return dict(peer=0, help=0) # def absolute_filename(*pargs, **kwargs): -# return server.absolute_filename(*pargs, **kwargs) +# return absolute_filename(*pargs, **kwargs) -def update_server(**kwargs): - for arg, func in kwargs.items(): - # logmessage("Setting " + str(arg)) - if arg == 'bg_action': - def worker_wrapper(action, ui_notification, the_func=func, **kwargs): - return worker_caller(the_func, ui_notification, {'action': action, 'arguments': kwargs}) - setattr(server, arg, worker_wrapper) - else: - setattr(server, arg, func) +# def update_server(**kwargs): +# for arg, func in kwargs.items(): +# # logmessage("Setting " + str(arg)) +# if arg == 'bg_action': +# def worker_wrapper(action, ui_notification, the_func=func, **kwargs): +# return worker_caller(the_func, ui_notification, {'action': action, 'arguments': kwargs}) +# setattr(server, arg, worker_wrapper) +# else: +# setattr(server, arg, func) # the_write_record = basic_write_record @@ -2447,10 +2533,10 @@ def __init__(self): self.role = 'user' # class ThreadVariables(threading.local): -# language = server.default_language -# dialect = server.default_dialect -# country = server.default_country -# locale = server.default_locale +# language = get_default_language() +# dialect = get_default_dialect() +# country = get_default_country() +# locale = get_default_locale() # current_info = {} # internal = {} # # user_dict = None @@ -2481,75 +2567,75 @@ def __init__(self): # self.initialized = True # self.__dict__.update(kw) -this_thread = threading.local() - -def populate_this_thread_defaults(): - this_thread.language = server.default_language - this_thread.dialect = server.default_dialect - this_thread.voice = server.default_voice - this_thread.country = server.default_country - this_thread.locale = server.default_locale - this_thread.current_info = {} - this_thread.internal = {} - this_thread.initialized = False - this_thread.session_id = None - this_thread.current_package = None - this_thread.interview = None - this_thread.interview_status = None - this_thread.evaluation_context = None - this_thread.gathering_mode = {} - this_thread.global_vars = GenericObject() - this_thread.current_variable = [] - this_thread.open_files = set() - this_thread.markdown = markdown.Markdown(extensions=['smarty', 'markdown.extensions.sane_lists', 'markdown.extensions.tables', 'markdown.extensions.attr_list', 'markdown.extensions.md_in_html', 'footnotes'], output_format='html5') - this_thread.saved_files = {} - this_thread.message_log = [] - this_thread.misc = {} - this_thread.probing = False - this_thread.prevent_going_back = False - this_thread.current_question = None - this_thread.current_section = None - -populate_this_thread_defaults() - - -def enable_threading(): - global this_thread - this_thread = SimpleNamespace() - populate_this_thread_defaults() - - -def backup_thread_variables(): - reset_context() - for key in ('pending_error', 'docx_subdocs', 'dbcache'): - if key in this_thread.misc: - del this_thread.misc[key] - backup = {} - for key in ('interview', 'interview_status', 'open_files', 'current_question'): - if hasattr(this_thread, key): - backup[key] = getattr(this_thread, key) - for key in ['language', 'dialect', 'country', 'locale', 'current_info', 'internal', 'initialized', 'session_id', 'current_package', 'interview', 'interview_status', 'evaluation_context', 'gathering_mode', 'global_vars', 'current_variable', 'saved_files', 'message_log', 'misc', 'probing', 'prevent_going_back', 'current_question']: - if hasattr(this_thread, key): - backup[key] = getattr(this_thread, key) - if key == 'global_vars': - this_thread.global_vars = GenericObject() - elif key == 'misc': - for key in [item for item in this_thread.misc.keys() if item.startswith('yaml_')]: - del this_thread.misc[key] - setattr(this_thread, key, copy.deepcopy(this_thread.misc)) - elif key == 'current_info': - setattr(this_thread, key, copy.deepcopy(getattr(this_thread, key))) - elif key in ('internal', 'gathering_mode', 'saved_files'): - setattr(this_thread, key, {}) - elif key in ('current_variable', 'message_log'): - setattr(this_thread, key, []) - return backup - - -def restore_thread_variables(backup): - # logmessage("restore_thread_variables") - for key in list(backup.keys()): - setattr(this_thread, key, backup[key]) + +# exec with user_dict +# docassemble_base/docassemble/base/parse.py +# docassemble_base/docassemble/base/functions.py +# docassemble_webapp/docassemble/webapp/tasks/worker_tasks.py +# docassemble_webapp/docassemble/webapp/interview/helpers.py +# docassemble_webapp/docassemble/webapp/interview/views.py +# docassemble_webapp/docassemble/webapp/sms/views.py + +# reset_local_variables: +# docassemble_base/docassemble/base/parse.py +# - commented out +# docassemble_base/docassemble/base/functions.py +# - function definition +# docassemble_webapp/docassemble/webapp/cron.py +# - before running assemble +# docassemble_webapp/docassemble/webapp/tasks/worker_tasks.py +# - within each task +# docassemble_webapp/docassemble/webapp/tasks/worker_common.py +# - bg_context contextmanager +# docassemble_webapp/docassemble/webapp/main/views.py +# - before_request + +# backup_thread_variables: +# docassemble_base/docassemble/base/functions.py +# - function definition +# docassemble_webapp/docassemble/webapp/interview/helpers.py +# - get_session_variables +# - go_back_in_session +# - set_session_variables +# - create_new_interview +# - get_question_data +# - run_action_in_session +# docassemble_webapp/docassemble/webapp/sms/helpers.py +# - sms_body, before calling do_sms. Called by send_sms_invite + + + +# def backup_thread_variables(): +# reset_context() +# for key in ('pending_error', 'docx_subdocs', 'dbcache'): +# if key in this_thread.misc: +# del this_thread.misc[key] +# backup = {} +# for key in ('interview', 'interview_status', 'open_files', 'current_question'): +# if hasattr(this_thread, key): +# backup[key] = getattr(this_thread, key) +# for key in ['language', 'dialect', 'country', 'locale', 'current_info', 'internal', 'initialized', 'session_id', 'current_package', 'interview', 'interview_status', 'evaluation_context', 'gathering_mode', 'global_vars', 'current_variable', 'saved_files', 'message_log', 'misc', 'probing', 'prevent_going_back', 'current_question']: +# if hasattr(this_thread, key): +# backup[key] = getattr(this_thread, key) +# if key == 'global_vars': +# this_thread.global_vars = GenericObject() +# elif key == 'misc': +# for key in [item for item in this_thread.misc.keys() if item.startswith('yaml_')]: +# del this_thread.misc[key] +# setattr(this_thread, key, copy.deepcopy(this_thread.misc)) +# elif key == 'current_info': +# setattr(this_thread, key, copy.deepcopy(getattr(this_thread, key))) +# elif key in ('internal', 'gathering_mode', 'saved_files'): +# setattr(this_thread, key, {}) +# elif key in ('current_variable', 'message_log'): +# setattr(this_thread, key, []) +# return backup + + +# def restore_thread_variables(backup): +# # logmessage("restore_thread_variables") +# for key in list(backup.keys()): +# setattr(this_thread, key, backup[key]) def background_response(*pargs, **kwargs): @@ -2619,69 +2705,19 @@ def background_action(*pargs, **kwargs): ui_notification = pargs[1] else: ui_notification = None - return server.bg_action(action, ui_notification, **kwargs) - - -class BackgroundResult: - - def __init__(self, result): - for attr in ('value', 'error_type', 'error_trace', 'error_message', 'variables'): - if hasattr(result, attr): - setattr(self, attr, getattr(result, attr)) - else: - setattr(self, attr, None) - - -class MyAsyncResult: - - def wait(self): - if not hasattr(self, '_cached_result'): - self._cached_result = BackgroundResult(server.worker_convert(self.obj).get()) - return True - - def failed(self): - if not hasattr(self, '_cached_result'): - self._cached_result = BackgroundResult(server.worker_convert(self.obj).get()) - if self._cached_result.error_type is not None: - return True - return False - - def ready(self): - return server.worker_convert(self.obj).ready() - - def result(self): - if not hasattr(self, '_cached_result'): - self._cached_result = BackgroundResult(server.worker_convert(self.obj).get()) - return self._cached_result - - def get(self): - if not hasattr(self, '_cached_result'): - self._cached_result = BackgroundResult(server.worker_convert(self.obj).get()) - return self._cached_result.value - - def revoke(self, terminate=True): - return server.worker_convert(self.obj).revoke(terminate=terminate) + return bg_action(action, ui_notification, **kwargs) - def status(self): - return server.worker_convert(self.obj).status - def state(self): - return server.worker_convert(self.obj).state - - def date_done(self): - return server.worker_convert(self.obj).date_done - - -def worker_caller(func, ui_notification, action): - # logmessage("Got to worker_caller in functions") - result = MyAsyncResult() - result.obj = func.delay(this_thread.current_info['yaml_filename'], this_thread.current_info['user'], this_thread.current_info['session'], this_thread.current_info['secret'], this_thread.current_info['url'], this_thread.current_info['url_root'], action, extra=ui_notification) - if ui_notification is not None: - worker_key = 'da:worker:uid:' + str(this_thread.current_info['session']) + ':i:' + str(this_thread.current_info['yaml_filename']) + ':userid:' + str(this_thread.current_info['user']['the_user_id']) - # logmessage("worker_caller: id is " + str(result.obj.id) + " and key is " + worker_key) - server.server_redis.rpush(worker_key, result.obj.id) - # logmessage("worker_caller: id is " + str(result.obj.id)) - return result +# def worker_caller(func, ui_notification, action): +# # logmessage("Got to worker_caller in functions") +# result = MyAsyncResult() +# result.obj = func.delay(this_thread.current_info['yaml_filename'], this_thread.current_info['user'], this_thread.current_info['session'], this_thread.current_info['secret'], this_thread.current_info['url'], this_thread.current_info['url_root'], action, extra=ui_notification) +# if ui_notification is not None: +# worker_key = 'da:worker:uid:' + str(this_thread.current_info['session']) + ':i:' + str(this_thread.current_info['yaml_filename']) + ':userid:' + str(this_thread.current_info['user']['the_user_id']) +# # logmessage("worker_caller: id is " + str(result.obj.id) + " and key is " + worker_key) +# get_server_redis().rpush(worker_key, result.obj.id) +# # logmessage("worker_caller: id is " + str(result.obj.id)) +# return result # def null_chat_partners(*pargs, **kwargs): # return dict(peer=0, help=0) @@ -2760,58 +2796,6 @@ def dump_to_bytes(self, *pargs, **kwargs): altyamlstring = SafeYaml('yaml_altyamlstring') -def ordinal_function_en(i, **kwargs): - try: - i = int(i) - except: - i = 0 - use_word = kwargs.get('use_word', None) - if use_word is True: - kwargs['function'] = 'ordinal' - elif use_word is False: - kwargs['function'] = 'ordinal_num' - else: - if i < 11: - kwargs['function'] = 'ordinal' - else: - kwargs['function'] = 'ordinal_num' - return number_to_word(i, **kwargs) - -ordinal_functions = { - 'en': ordinal_function_en, - '*': ordinal_function_en -} - - -def fix_punctuation(text, mark=None, other_marks=None): - """Ensure the text ends with a punctuation mark, adding one if necessary. - - Args: - text (str): The text to check. - mark (str, optional): The punctuation mark to append if none is - present. Defaults to ``'.'``. - other_marks (list, optional): A list of punctuation marks that are - considered acceptable endings. Defaults to ``['.', '?', '!']``. - - Returns: - str: The text, possibly with a punctuation mark appended. - """ - ensure_definition(text, mark, other_marks) - if other_marks is None: - other_marks = ['.', '?', '!'] - if not isinstance(other_marks, list): - other_marks = list(other_marks) - if mark is None: - mark = '.' - text = text.rstrip() - if mark == '': - return text - for end_mark in set([mark] + other_marks): - if text.endswith(end_mark): - return text - return text + mark - - def item_label(num, level=None, punctuation=True): """Return a formatted list item label for a given zero-based index and outline level. @@ -2919,1902 +2903,131 @@ def roman(num, case=None): return result -def words(): - return word_collection[this_thread.language] - - -class LazyWord: - - def __init__(self, *args, **kwargs): - if len(kwargs) > 0: - self.original = args[0] % kwargs - else: - self.original = args[0] - - def __mod__(self, other): - return word(self.original) % other - - def __str__(self): - return word(self.original) - - -class LazyArray: - - def __init__(self, array): - self.original = array - - def compute(self): - return [word(item) for item in self.original] - - def copy(self): - return self.compute().copy() - - def pop(self, *pargs): - return str(self.original.pop(*pargs)) - - def __add__(self, other): - return self.compute() + other - - def index(self, *pargs, **kwargs): - return self.compute().index(*pargs, **kwargs) - - def clear(self): - self.original = [] - - def append(self, other): - self.original.append(other) - - def remove(self, other): - self.original.remove(other) - - def extend(self, other): - self.original.extend(other) - - def __contains__(self, item): - return self.compute().__contains__(item) - - def __iter__(self): - return self.compute().__iter__() - - def __len__(self): - return self.compute().__len__() - - def __delitem__(self, index): - self.original.__delitem__(index) - - def __reversed__(self): - return self.compute().__reversed__() - - def __setitem__(self, index, the_value): - return self.original.__setitem__(index, the_value) - - def __getitem__(self, index): - return self.compute()[index] - - def __str__(self): - return str(self.compute()) - - def __repr__(self): - return repr(self.compute()) - - def __eq__(self, other): - return self.original == other - - -def word(the_word, **kwargs): - """Return the word translated into the current language. - - If no translation is found for the current language, the input is - returned unchanged. Used throughout docassemble to support - multilingual interviews. - - Args: - the_word (str): The word or phrase to translate. - **kwargs: Optional keyword arguments. Pass ``language`` to - look up a translation for a specific language, or - ``capitalize=True`` to capitalize the result. - - Returns: - str: The translated (or original) word. - """ - # Currently, no kwargs are used, but in the future, this function could be - # expanded to use kwargs. For example, for languages with gendered words, - # the gender could be passed as a keyword argument. - if the_word is True: - the_word = 'yes' - elif the_word is False: - the_word = 'no' - elif the_word is None: - the_word = "I don't know" - if isinstance(the_word, LazyWord): - the_word = the_word.original - try: - the_word = word_collection[kwargs.get('language', this_thread.language)][the_word] - except: - the_word = str(the_word) - if kwargs.get('capitalize', False): - return capitalize(the_word) - return the_word - - -def update_language_function(lang, term, func): - if term not in language_functions: - language_functions[term] = {} - language_functions[term][lang] = func - - -def update_nice_numbers(lang, defs): - if lang not in nice_numbers: - nice_numbers[lang] = {} - for number, the_word in defs.items(): - nice_numbers[lang][str(number)] = the_word - - -def update_ordinal_numbers(lang, defs): - if lang not in ordinal_numbers: - ordinal_numbers[lang] = {} - for number, the_word in defs.items(): - ordinal_numbers[lang][str(number)] = the_word - - -def update_ordinal_function(lang, func): - ordinal_functions[lang] = func - - -def update_word_collection(lang, defs): - if lang not in word_collection: - word_collection[lang] = {} - for the_word, translation in defs.items(): - if translation is not None: - word_collection[lang][the_word] = translation - -# def set_da_config(config): -# global daconfig -# daconfig = config - - -def get_config(key, none_value=None): - """Return a value from the docassemble configuration file. - - Args: - key (str): The configuration directive to look up. - none_value (optional): The value to return if the key is not found in - the configuration. Defaults to None. - - Returns: - The configuration value associated with the key, or ``none_value`` - if the key is not present. - """ - return server.daconfig.get(key, none_value) - -# def set_default_language(lang): -# global default_language -# default_language = lang - -# def set_default_dialect(dialect): -# global default_dialect -# default_dialect = dialect -# return - -# def set_default_country(country): -# global default_country -# default_country = country -# return - -# def set_default_timezone(timezone): -# global default_timezone -# default_timezone = timezone -# return - - -def get_default_timezone(): - """Return the default timezone string for the server. - - Returns the server's local timezone unless a default timezone is configured - in the docassemble configuration. - - Returns: - str: A timezone string such as ``'America/New_York'``. - """ - return server.default_timezone - -# def reset_thread_local(): -# this_thread.open_files = set() -# this_thread.temporary_resources = set() - -# def reset_thread_variables(): -# this_thread.saved_files = {} -# this_thread.message_log = [] - - -def reset_local_variables(): - # logmessage("reset_local_variables") - this_thread.language = server.default_language - this_thread.dialect = server.default_dialect - this_thread.voice = server.default_voice - this_thread.country = server.default_country - this_thread.locale = server.default_locale - this_thread.session_id = None - this_thread.interview = None - this_thread.interview_status = None - this_thread.evaluation_context = None - this_thread.gathering_mode = {} - this_thread.global_vars = GenericObject() - this_thread.current_variable = [] - # this_thread.template_vars = [] - this_thread.open_files = set() - this_thread.saved_files = {} - this_thread.message_log = [] - this_thread.misc = {} - this_thread.probing = False - this_thread.current_info = {} - this_thread.current_package = None - this_thread.current_question = None - this_thread.current_section = None - this_thread.internal = {} - this_thread.markdown = markdown.Markdown(extensions=['smarty', 'markdown.extensions.sane_lists', 'markdown.extensions.tables', 'markdown.extensions.attr_list', 'markdown.extensions.md_in_html', 'footnotes'], output_format='html5') - this_thread.prevent_going_back = False - - -def prevent_going_back(): - """Disable the back button so the user cannot revisit previous questions. - - Once called, the user will not be able to go back and change any answers - entered before this point in the interview. - """ - this_thread.prevent_going_back = True - - -def set_language(lang, dialect=None, voice=None): - """Set the language used for linguistic functions and the web application. - - Does not change the Python locale; call ``update_locale()`` for that. - Should be called in an ``initial`` code block so it takes effect on every - page load. - - Args: - lang (str): A lowercase ISO-639-1 or ISO-639-3 language code - (e.g., ``'en'``, ``'es'``, ``'fr'``). - dialect (str, optional): A dialect code for the text-to-speech engine. - Defaults to None. - voice (str, optional): A voice name for the text-to-speech engine. - Defaults to None. - """ - try: - if dialect: - this_thread.dialect = dialect - elif lang != this_thread.language: - this_thread.dialect = None - except: - pass - try: - if voice: - this_thread.voice = voice - elif lang != this_thread.language: - this_thread.voice = None - except: - pass - this_thread.language = lang - - -def get_language(): - """Return the current language code. - - Returns: - str: The current language code (e.g., ``'en'``, ``'es'``). - """ - return this_thread.language - - -def set_country(country): - """Set the current country used for phone number formatting and other locale features. - - Args: - country (str): A two-letter uppercase ISO 3166-1 alpha-2 country code - (e.g., ``'US'``, ``'GB'``, ``'DE'``). - """ - this_thread.country = country - - -def get_country(): - """Return the current country code. - - Returns: - str: A two-letter uppercase ISO 3166-1 alpha-2 country code - (e.g., ``'US'``). Defaults to ``'US'`` unless configured otherwise. - """ - return this_thread.country - - -def get_dialect(): - """Return the current dialect. - - Returns: - str: The dialect code set by the ``dialect`` keyword argument to - :func:`set_language`, or ``None`` if no dialect has been set. - """ - return this_thread.dialect - - -def get_voice(): - """Return the current voice. - - Returns: - str: The voice name set by the ``voice`` keyword argument to - :func:`set_language`, or ``None`` if no voice has been set. - """ - return this_thread.voice - - -def set_locale(*pargs, **kwargs): - """Set the current locale string and/or locale convention overrides. - - Calling ``set_locale('FR.utf8')`` stores the locale string so that - :func:`get_locale` returns it. The actual Python locale does not change - until :func:`update_locale` is called. Keyword arguments such as - ``currency_symbol`` override individual locale conventions used by - functions like :func:`currency` and :func:`currency_symbol`. - - Args: - *pargs: An optional locale string (e.g. ``'FR.utf8'``). - **kwargs: Locale convention overrides (e.g. ``currency_symbol='€'``). - """ - if len(pargs) == 1: - this_thread.locale = pargs[0] - if len(kwargs): - this_thread.misc['locale_overrides'] = kwargs - - -def get_locale(*pargs): - """Return the current locale setting or a specific locale convention. - - With no arguments, returns the locale string previously set with - :func:`set_locale`. With one argument, returns the value of the named - locale convention (e.g. ``'currency_symbol'``), taking into account any - overrides set with :func:`set_locale`. - - Args: - *pargs: An optional locale convention name (e.g. - ``'currency_symbol'``). - - Returns: - str or None: The locale string when called with no arguments, or the - value of the requested locale convention (``None`` if not found). - """ - if len(pargs) == 1: - if 'locale_overrides' in this_thread.misc and pargs[0] in this_thread.misc['locale_overrides']: - return this_thread.misc['locale_overrides'][pargs[0]] - return locale.localeconv().get(pargs[0], None) - return this_thread.locale - - -def get_currency_symbol(): - """Returns the current setting for the currency symbol if there is - one, and otherwise returns the default currency symbol. - - """ - if 'locale_overrides' in this_thread.misc and 'currency_symbol' in this_thread.misc['locale_overrides']: - return this_thread.misc['locale_overrides']['currency_symbol'] - return currency_symbol() - - -def update_locale(): - """Update the Python locale based on the current language and locale settings. - - Applies the locale string previously set with :func:`set_locale` (combined - with the current language from :func:`get_language` when necessary) so that - Python's ``locale`` module reflects the desired locale. This is required - for functions like :func:`currency` and :func:`currency_symbol` to produce - locale-appropriate formatting. - """ - if '_' in this_thread.locale: - the_locale = str(this_thread.locale) - else: - the_locale = str(this_thread.language) + '_' + str(this_thread.locale) - try: - locale.setlocale(locale.LC_ALL, the_locale) - except BaseException as err: - logmessage("update_locale error: unable to set the locale to " + the_locale) - logmessage(err.__class__.__name__ + ": " + str(err)) - locale.setlocale(locale.LC_ALL, 'en_US.utf8') - - -def comma_list_en(*pargs, **kwargs): - """Returns the arguments separated by commas. If the first argument is a list, - that list is used. Otherwise, the arguments are treated as individual items. - See also comma_and_list().""" - ensure_definition(*pargs, **kwargs) - comma_string = kwargs.get('comma_string', ', ') - the_list = [] - for parg in pargs: - if isinstance(parg, str): - the_list.append(parg) - elif (hasattr(parg, 'instanceName') and hasattr(parg, 'elements')) or isinstance(parg, Iterable): - for sub_parg in parg: - the_list.append(str(sub_parg)) - else: - the_list.append(str(parg)) - return comma_string.join(the_list) - - -def comma_and_list_es(*pargs, **kwargs): - if 'and_string' not in kwargs: - kwargs['and_string'] = 'y' - return comma_and_list_en(*pargs, **kwargs) - - -def comma_and_list_de(*pargs, **kwargs): - if 'and_string' not in kwargs: - kwargs['and_string'] = 'und' - if 'oxford' not in kwargs: - kwargs['oxford'] = False - return comma_and_list_en(*pargs, **kwargs) - - -def comma_and_list_en(*pargs, **kwargs): - """Returns an English-language listing of the arguments. If the first argument is a list, - that list is used. Otherwise, the arguments are treated as individual items in the list. - Use the optional argument oxford=False if you do not want a comma before the "and." - See also comma_list().""" - ensure_definition(*pargs, **kwargs) - and_string = kwargs.get('and_string', word('and')) - comma_string = kwargs.get('comma_string', ', ') - if 'oxford' in kwargs and kwargs['oxford'] is False: - extracomma = "" - else: - extracomma = comma_string.strip() - before_and = kwargs.get('before_and', ' ') - after_and = kwargs.get('after_and', ' ') - the_list = [] - for parg in pargs: - if isinstance(parg, str): - the_list.append(parg) - elif (hasattr(parg, 'instanceName') and hasattr(parg, 'elements')) or isinstance(parg, Iterable): - for sub_parg in parg: - the_list.append(str(sub_parg)) - else: - the_list.append(str(parg)) - if len(the_list) == 0: - return str('') - if len(the_list) == 1: - return the_list[0] - if len(the_list) == 2: - return the_list[0] + before_and + and_string + after_and + the_list[1] - return comma_string.join(the_list[:-1]) + extracomma + before_and + and_string + after_and + the_list[-1] - - -def manual_line_breaks(text): - """Replaces newlines with manual line breaks.""" - if this_thread.evaluation_context == 'docx': - return re.sub(r' *\r?\n *', '', str(text)) - return re.sub(r' *\r?\n *', ' [BR] ', str(text)) - - -def add_separators_en(*pargs, **kwargs): - """Accepts a list and returns a list, with semicolons after each item, - except "and" after the penultimate item and a period after the - last. - - """ - ensure_definition(*pargs, **kwargs) - separator = kwargs.get('separator', ';') - last_separator = kwargs.get('last_separator', '; ' + word("and")) - end_mark = kwargs.get('end_mark', '.') - the_list = [] - for parg in pargs: - if isinstance(parg, str): - the_list.append(parg.rstrip()) - elif (hasattr(parg, 'instanceName') and hasattr(parg, 'elements')) or isinstance(parg, Iterable): - for sub_parg in parg: - the_list.append(str(sub_parg).rstrip()) - else: - the_list.append(str(parg).rstrip()) - if len(the_list) == 0: - return the_list - if len(the_list) == 1: - return [fix_punctuation(the_list[0], mark=end_mark)] - for indexno in range(len(the_list) - 2): # for 4: 0, 1; for 3: 0; for 2: [] - the_list[indexno] = the_list[indexno].rstrip(',') - the_list[indexno] = fix_punctuation(the_list[indexno], mark=separator) - if not the_list[-2].endswith(last_separator): - the_list[-2] = the_list[-2].rstrip(last_separator[0]) - the_list[-2] += last_separator - the_list[-1] = fix_punctuation(the_list[-1], mark=end_mark) - return the_list - - -def need(*pargs): - """Ensure that the given variables are defined, asking questions if necessary. - - Evaluating each argument causes docassemble to seek its definition - through the normal interview logic. The function always returns - ``True``. Using ``need()`` is purely for readability; writing - ``need(x, y)`` is equivalent to writing ``x; y`` in a code block. - - Args: - *pargs: Variables whose definitions should be ensured. - - Returns: - bool: Always ``True``. - """ - ensure_definition(*pargs) - for argument in pargs: - argument # pylint: disable=pointless-statement - return True - - -def pickleable_objects(input_dict): - output_dict = {} - for key in input_dict: - if isinstance(input_dict[key], (types.ModuleType, types.FunctionType, TypeType, types.BuiltinFunctionType, types.BuiltinMethodType, types.MethodType, FileType)): - continue - if key == "__builtins__": - continue - output_dict[key] = input_dict[key] - return output_dict - - -def ordinal_number_default(the_number, **kwargs): - """Returns the "first," "second," "third," etc. for a given number. - ordinal_number(1) returns "first." For a function that can be used - on index numbers that start with zero, see ordinal().""" - num = str(the_number) - if kwargs.get('use_word', True): - if this_thread.language in ordinal_numbers and num in ordinal_numbers[this_thread.language]: - return ordinal_numbers[this_thread.language][num] - if '*' in ordinal_numbers and num in ordinal_numbers['*']: - return ordinal_numbers['*'][num] - if this_thread.language in ordinal_functions: - language_to_use = this_thread.language - elif '*' in ordinal_functions: - language_to_use = '*' - else: - language_to_use = 'en' - return ordinal_functions[language_to_use](the_number, **kwargs) - - -def salutation_default(indiv, **kwargs): - """Returns Mr., Ms., etc. for an individual.""" - with_name = kwargs.get('with_name', False) - with_name_and_punctuation = kwargs.get('with_name_and_punctuation', False) - ensure_definition(indiv, with_name, with_name_and_punctuation) - used_gender = False - if hasattr(indiv, 'salutation_to_use') and indiv.salutation_to_use is not None: - salut = indiv.salutation_to_use - elif hasattr(indiv, 'is_doctor') and indiv.is_doctor: - salut = 'Dr.' - elif hasattr(indiv, 'is_judge') and indiv.is_judge: - salut = 'Judge' - elif hasattr(indiv, 'name') and hasattr(indiv.name, 'suffix') and indiv.name.suffix in ('MD', 'PhD'): - salut = 'Dr.' - elif hasattr(indiv, 'name') and hasattr(indiv.name, 'suffix') and indiv.name.suffix == 'J': - salut = 'Judge' - elif indiv.gender == 'female': - used_gender = True - salut = 'Ms.' - else: - used_gender = True - salut = 'Mr.' - if with_name_and_punctuation or with_name: - if used_gender and indiv.gender not in ('male', 'female'): - salut_and_name = indiv.name.full() - else: - salut_and_name = salut + ' ' + indiv.name.last - if with_name_and_punctuation: - if hasattr(indiv, 'is_friendly') and indiv.is_friendly: - punct = ',' - else: - punct = ':' - return salut_and_name + punct - if with_name: - return salut_and_name - return salut - - -def string_to_number(number): - try: - float_number = float(number) - int_number = int(number) - if float_number == int_number: - return int_number - return float_number - except: - return number - - -def number_to_word(number, **kwargs): - language = kwargs.get('language', None) - capitalize_arg = kwargs.get('capitalize', False) - function = kwargs.get('function', None) - raise_on_error = kwargs.get('raise_on_error', False) - if function not in ('ordinal', 'ordinal_num'): - function = 'cardinal' - if language is None: - language = get_language() - for lang, loc in (('en', 'en_GB'), ('en', 'en_IN'), ('es', 'es_CO'), ('es', 'es_VE'), ('fr', 'fr_CH'), ('fr', 'fr_BE'), ('fr', 'fr_DZ'), ('pt', 'pt_BR')): - if language == lang and this_thread.locale.startswith(loc): - language = loc - break - number = string_to_number(number) - if raise_on_error: - the_word = num2words.num2words(number, lang=language, to=function) - else: - try: - the_word = num2words.num2words(number, lang=language, to=function) - except NotImplementedError: - the_word = str(number) - if capitalize_arg: - return capitalize_function(the_word) - return the_word - - -def ordinal_default(the_number, **kwargs): - """Returns the "first," "second," "third," etc. for a given number, which is expected to - be an index starting with zero. ordinal(0) returns "first." For a more literal ordinal - number function, see ordinal_number().""" - result = ordinal_number(int(float(the_number)) + 1, **kwargs) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(result) - return result - - -def nice_number_default(the_number, **kwargs): - """Returns the number as a word in the current language.""" - capitalize_arg = kwargs.get('capitalize', False) - language = kwargs.get('language', None) - use_word = kwargs.get('use_word', None) - ensure_definition(the_number, capitalize_arg, language) - if language is None: - language = this_thread.language - if language in nice_numbers: - language_to_use = language - elif '*' in nice_numbers: - language_to_use = '*' - else: - language_to_use = 'en' - if isinstance(the_number, float): - the_number = float(decimal.Context(prec=8).create_decimal_from_float(the_number)) - if int(float(the_number)) == float(the_number): - the_number = int(float(the_number)) - is_integer = True - else: - is_integer = False - if language_to_use in nice_numbers and str(the_number) in nice_numbers[language_to_use]: - the_word = nice_numbers[language_to_use][str(the_number)] - if capitalize_arg: - return capitalize_function(the_word) - return the_word - if use_word or (is_integer and 0 <= the_number < 11 and use_word is not False): - try: - return number_to_word(the_number, **kwargs) - except: - pass - if isinstance(the_number, int): - return str(locale.format_string("%d", the_number, grouping=True)) - return str(locale.format_string("%.2f", float(the_number), grouping=True)).rstrip('0') - - -def quantity_noun_default(the_number, noun, **kwargs): - as_integer = kwargs.get('as_integer', True) - capitalize_arg = kwargs.get('capitalize', False) - language = kwargs.get('language', None) - ensure_definition(the_number, noun, as_integer, capitalize_arg, language) - if as_integer: - the_number = int(round(the_number)) - result = nice_number(the_number, language=language) + " " + noun_plural(noun, the_number, language=language) - if capitalize_arg: - return capitalize_function(result) - return result - - -def capitalize_default(a, **kwargs): # pylint: disable=unused-argument - ensure_definition(a) - if not isinstance(a, str): - a = str(a) - if a and len(a) > 1: - return a[0].upper() + a[1:] - return a - - -def currency_symbol_default(**kwargs): # pylint: disable=unused-argument - """Returns the currency symbol for the current locale.""" - return str(locale.localeconv()['currency_symbol']) - - -def currency_default(the_value, **kwargs): - """Returns the value as a currency, according to the conventions of - the current locale. Use the optional keyword argument - decimals=False if you do not want to see decimal places in the - number, and the optional currency_symbol for a different symbol - than the default. - - """ - decimals = kwargs.get('decimals', True) - symbol = kwargs.get('symbol', None) - symbol_precedes = kwargs.get('symbol_precedes', None) - ensure_definition(the_value, decimals, symbol) - obj_type = type(the_value).__name__ - if obj_type in ['FinancialList', 'PeriodicFinancialList']: - the_value = the_value.total() - elif obj_type in ['Value', 'PeriodicValue']: - if the_value.exists: - the_value = the_value.amount() - else: - the_value = 0 - elif obj_type == 'DACatchAll': - the_value = float(the_value) - try: - float(the_value) - except: - return '' - the_float_value = float(the_value) - the_symbol = None - if symbol is not None: - the_symbol = symbol - elif 'locale_overrides' in this_thread.misc and 'currency_symbol' in this_thread.misc['locale_overrides']: - the_symbol = this_thread.misc['locale_overrides']['currency_symbol'] - elif language_functions['currency_symbol']['*'] is not currency_symbol_default: - the_symbol = currency_symbol() - the_symbol_precedes = None - if symbol_precedes is not None: - the_symbol_precedes = symbol_precedes - elif 'locale_overrides' in this_thread.misc and the_float_value < 0 and 'n_cs_precedes' in this_thread.misc['locale_overrides']: - the_symbol_precedes = bool(this_thread.misc['locale_overrides']['n_cs_precedes']) - elif 'locale_overrides' in this_thread.misc and 'p_cs_precedes' in this_thread.misc['locale_overrides']: - the_symbol_precedes = bool(this_thread.misc['locale_overrides']['p_cs_precedes']) - if the_symbol is None and the_symbol_precedes is None and decimals: - return str(locale.currency(the_float_value, symbol=True, grouping=True)) - if the_symbol is None: - the_symbol = currency_symbol() - if the_symbol_precedes is None: - if the_float_value < 0: - the_symbol_precedes = bool(get_locale('n_cs_precedes')) - else: - the_symbol_precedes = bool(get_locale('p_cs_precedes')) - output = '' - if the_symbol_precedes: - output += the_symbol - if the_float_value < 0: - if get_locale('n_sep_by_space'): - output += ' ' - elif get_locale('p_sep_by_space'): - output += ' ' - if decimals: - output += locale.format_string('%.' + str(server.daconfig.get('currency decimal places', locale.localeconv()['frac_digits'])) + 'f', the_float_value, grouping=True, monetary=True) - else: - output += locale.format_string("%d", int(the_float_value), grouping=True, monetary=True) - if not the_symbol_precedes: - if the_float_value < 0: - if get_locale('n_sep_by_space'): - output += ' ' - elif get_locale('p_sep_by_space'): - output += ' ' - output += the_symbol - return output - - -def prefix_constructor(prefix): - - def func(the_word, **kwargs): - ensure_definition(the_word, **kwargs) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(str(prefix)) + str(the_word) - return str(prefix) + str(the_word) - return func - - -def double_prefix_constructor_reverse(prefix_one, prefix_two): - - def func(word_one, word_two, **kwargs): - ensure_definition(word_one, word_two, **kwargs) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(str(prefix_one)) + str(word_two) + str(prefix_two) + str(word_one) - return str(prefix_one) + str(word_two) + str(prefix_two) + str(word_one) - return func - - -def prefix_constructor_two_arguments(prefix, **kwargs): # pylint: disable=unused-argument - - def func(word_one, word_two, **kwargs): - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(str(prefix)) + str(word_one) + ' ' + str(word_two) - return str(prefix) + str(word_one) + ' ' + str(word_two) - return func - - -def middle_constructor(middle, **kwargs): # pylint: disable=unused-argument - - def func(a, b, **kwargs): - ensure_definition(a, b, **kwargs) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(str(a)) + str(middle) + str(b) - return str(a) + str(middle) + str(b) - return func - - -def possessify_en(a, b, **kwargs): - ensure_definition(a, b, **kwargs) - if this_thread.evaluation_context == 'docx': - apostrophe = "’" - else: - apostrophe = "'" - if 'plural' in kwargs and kwargs['plural']: - middle = apostrophe + " " - else: - middle = apostrophe + "s " - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(str(a)) + str(middle) + str(b) - return str(a) + str(middle) + str(b) - - -def a_preposition_b_default(a, b, **kwargs): - ensure_definition(a, b, **kwargs) - if hasattr(a, 'preposition'): - preposition = word(a.preposition) - else: - preposition = word('in the') - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(str(a)) + str(' ' + preposition + ' ') + str(b) - return str(a) + str(' ' + preposition + ' ') + str(b) - - -def verb_present_en(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - new_args = [] - for arg in pargs: - new_args.append(str(arg)) - if len(new_args) < 2: - new_args.append('3sg') - output = pattern_en.conjugate(*new_args, **kwargs) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def verb_past_en(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - new_args = [] - for arg in pargs: - new_args.append(arg) - if len(new_args) < 2: - new_args.append('3sgp') - output = pattern_en.conjugate(*new_args, **kwargs) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def number_or_length(target): - if isinstance(target, (int, float)): - return target - if isinstance(target, (list, dict, set, tuple)) or (hasattr(target, 'elements') and isinstance(target.elements, (list, dict, set))): - return len(target) - if target: - return 2 - return 1 - - -def noun_plural_en(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - if kwargs.get('noun_is_singular', False): - noun = pargs[0] - else: - noun = noun_singular_en(pargs[0]) - if len(pargs) >= 2 and number_or_length(pargs[1]) == 1: - return str(noun) - output = pattern_en.pluralize(str(noun)) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def noun_singular_en(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - if len(pargs) >= 2 and number_or_length(pargs[1]) != 1: - return pargs[0] - output = pattern_en.singularize(str(pargs[0])) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def indefinite_article_en(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - output = pattern_en.article(str(pargs[0]).lower()) + " " + str(pargs[0]) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def verb_present_es(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - new_args = [] - for arg in pargs: - new_args.append(str(arg)) - if len(new_args) < 2: - new_args.append('3sg') - if new_args[1] == 'pl': - new_args[1] = '3pl' - output = pattern_es.conjugate(*new_args, **kwargs) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def verb_past_es(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - new_args = [] - for arg in pargs: - new_args.append(arg) - if len(new_args) < 2: - new_args.append('3sgp') - if new_args[1] == 'ppl': - new_args[1] = '3ppl' - output = pattern_es.conjugate(*new_args, **kwargs) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def noun_plural_es(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - if kwargs.get('noun_is_singular', False): - noun = pargs[0] - else: - noun = noun_singular_es(pargs[0]) - if len(pargs) >= 2 and number_or_length(pargs[1]) == 1: - return str(noun) - output = pattern_es.pluralize(str(noun)) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def noun_singular_es(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - if len(pargs) >= 2 and number_or_length(pargs[1]) != 1: - return pargs[0] - output = pattern_es.singularize(str(pargs[0])) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def indefinite_article_es(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - output = pattern_es.article(str(pargs[0]).lower()) + " " + str(pargs[0]) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def verb_present_de(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - new_args = [] - for arg in pargs: - new_args.append(str(arg)) - if len(new_args) < 2: - new_args.append('3sg') - if new_args[1] == 'pl': - new_args[1] = '3pl' - output = pattern_de.conjugate(*new_args, **kwargs) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def verb_past_de(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - new_args = [] - for arg in pargs: - new_args.append(arg) - if len(new_args) < 2: - new_args.append('3sgp') - if new_args[1] == 'ppl': - new_args[1] = '3ppl' - output = pattern_de.conjugate(*new_args, **kwargs) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def noun_plural_de(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - if kwargs.get('noun_is_singular', False): - noun = pargs[0] - else: - noun = noun_singular_de(pargs[0]) - if len(pargs) >= 2 and number_or_length(pargs[1]) == 1: - return str(noun) - output = pattern_de.pluralize(str(noun)) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def noun_singular_de(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - if len(pargs) >= 2 and number_or_length(pargs[1]) != 1: - return pargs[0] - output = pattern_de.singularize(str(pargs[0])) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def indefinite_article_de(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - output = pattern_de.article(str(pargs[0]).lower()) + " " + str(pargs[0]) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def verb_present_fr(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - new_args = [] - for arg in pargs: - new_args.append(str(arg)) - if len(new_args) < 2: - new_args.append('3sg') - if new_args[1] == 'pl': - new_args[1] = '3pl' - output = pattern_fr.conjugate(*new_args, **kwargs) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def verb_past_fr(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - new_args = [] - for arg in pargs: - new_args.append(arg) - if len(new_args) < 2: - new_args.append('3sgp') - if new_args[1] == 'ppl': - new_args[1] = '3ppl' - output = pattern_fr.conjugate(*new_args, **kwargs) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def noun_plural_fr(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - if kwargs.get('noun_is_singular', False): - noun = pargs[0] - else: - noun = noun_singular_fr(pargs[0]) - if len(pargs) >= 2 and number_or_length(pargs[1]) == 1: - return str(noun) - output = pattern_fr.pluralize(str(noun)) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def noun_singular_fr(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - if len(pargs) >= 2 and number_or_length(pargs[1]) != 1: - return pargs[0] - output = pattern_fr.singularize(str(pargs[0])) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def indefinite_article_fr(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - output = pattern_fr.article(str(pargs[0]).lower()) + " " + str(pargs[0]) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def verb_present_it(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - new_args = [] - for arg in pargs: - new_args.append(str(arg)) - if len(new_args) < 2: - new_args.append('3sg') - if new_args[1] == 'pl': - new_args[1] = '3pl' - output = pattern_it.conjugate(*new_args, **kwargs) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def verb_past_it(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - new_args = [] - for arg in pargs: - new_args.append(arg) - if len(new_args) < 2: - new_args.append('3sgp') - if new_args[1] == 'ppl': - new_args[1] = '3ppl' - output = pattern_it.conjugate(*new_args, **kwargs) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def noun_plural_it(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - if kwargs.get('noun_is_singular', False): - noun = pargs[0] - else: - noun = noun_singular_it(pargs[0]) - if len(pargs) >= 2 and number_or_length(pargs[1]) == 1: - return str(noun) - output = pattern_it.pluralize(str(noun)) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def noun_singular_it(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - if len(pargs) >= 2 and number_or_length(pargs[1]) != 1: - return pargs[0] - output = pattern_it.singularize(str(pargs[0])) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def indefinite_article_it(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - output = pattern_it.article(str(pargs[0]).lower()) + " " + str(pargs[0]) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def verb_present_nl(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - new_args = [] - for arg in pargs: - new_args.append(str(arg)) - if len(new_args) < 2: - new_args.append('3sg') - if new_args[1] == 'pl': - new_args[1] = '3pl' - output = pattern_nl.conjugate(*new_args, **kwargs) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def verb_past_nl(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - new_args = [] - for arg in pargs: - new_args.append(arg) - if len(new_args) < 2: - new_args.append('3sgp') - if new_args[1] == 'ppl': - new_args[1] = '3ppl' - output = pattern_nl.conjugate(*new_args, **kwargs) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def noun_plural_nl(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - if kwargs.get('noun_is_singular', False): - noun = pargs[0] - else: - noun = noun_singular_nl(pargs[0]) - if len(pargs) >= 2 and number_or_length(pargs[1]) == 1: - return str(noun) - output = pattern_nl.pluralize(str(noun)) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def noun_singular_nl(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - if len(pargs) >= 2 and number_or_length(pargs[1]) != 1: - return pargs[0] - output = pattern_nl.singularize(str(pargs[0])) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def indefinite_article_nl(*pargs, **kwargs): - ensure_definition(*pargs, **kwargs) - output = pattern_nl.article(str(pargs[0]).lower()) + " " + str(pargs[0]) - if 'capitalize' in kwargs and kwargs['capitalize']: - return capitalize(output) - return output - - -def titlecasestr(text): - return titlecase.titlecase(str(text)) - -language_functions = { - 'in_the': { - 'en': prefix_constructor('in the ') - }, - 'a_preposition_b': { - 'en': a_preposition_b_default - }, - 'a_in_the_b': { - 'en': middle_constructor(' in the ') - }, - 'i_subjective': { - 'en': lambda *pargs, **kwargs: word('I', **kwargs) - }, - 'he_subjective': { - 'en': lambda *pargs, **kwargs: word('he', **kwargs) - }, - 'she_subjective': { - 'en': lambda *pargs, **kwargs: word('she', **kwargs) - }, - 'genderless_subjective': { - 'en': lambda *pargs, **kwargs: word('they', **kwargs) - }, - 'myself': { - 'en': lambda *pargs, **kwargs: word('myself', **kwargs) - }, - 'itself': { - 'en': lambda *pargs, **kwargs: word('itself', **kwargs) - }, - 'herself': { - 'en': lambda *pargs, **kwargs: word('herself', **kwargs) - }, - 'himself': { - 'en': lambda *pargs, **kwargs: word('himself', **kwargs) - }, - 'themselves': { - 'en': lambda *pargs, **kwargs: word('themselves', **kwargs) - }, - 'genderless_self': { - 'en': lambda *pargs, **kwargs: word('themself', **kwargs) - }, - 'yourself': { - 'en': lambda *pargs, **kwargs: word('yourself', **kwargs) - }, - 'yourselves': { - 'en': lambda *pargs, **kwargs: word('yourselves', **kwargs) - }, - 'ourselves': { - 'en': lambda *pargs, **kwargs: word('ourselves', **kwargs) - }, - 'you_subjective': { - 'en': lambda *pargs, **kwargs: word('you', **kwargs) - }, - 'you_subjective_plural': { - 'en': lambda *pargs, **kwargs: word('you', **kwargs) - }, - 'we_subjective': { - 'en': lambda *pargs, **kwargs: word('we', **kwargs) - }, - 'they_subjective': { - 'en': lambda *pargs, **kwargs: word('they', **kwargs) - }, - 'it_subjective': { - 'en': lambda *pargs, **kwargs: word('it', **kwargs) - }, - 'it_objective': { - 'en': lambda *pargs, **kwargs: word('it', **kwargs) - }, - 'them_objective': { - 'en': lambda *pargs, **kwargs: word('them', **kwargs) - }, - 'genderless_objective': { - 'en': lambda *pargs, **kwargs: word('them', **kwargs) - }, - 'me_objective': { - 'en': lambda *pargs, **kwargs: word('me', **kwargs) - }, - 'him_objective': { - 'en': lambda *pargs, **kwargs: word('him', **kwargs) - }, - 'her_objective': { - 'en': lambda *pargs, **kwargs: word('her', **kwargs) - }, - 'you_objective': { - 'en': lambda *pargs, **kwargs: word('you', **kwargs) - }, - 'you_objective_plural': { - 'en': lambda *pargs, **kwargs: word('you', **kwargs) - }, - 'us_objective': { - 'en': lambda *pargs, **kwargs: word('us', **kwargs) - }, - 'are_we': { - 'en': lambda *pargs, **kwargs: word('are we', **kwargs) - }, - 'are_you': { - 'en': lambda *pargs, **kwargs: word('are you', **kwargs) - }, - 'are_you_plural': { - 'en': lambda *pargs, **kwargs: word('are you', **kwargs) - }, - 'am_i': { - 'en': lambda *pargs, **kwargs: word('am I', **kwargs) - }, - 'her': { - 'en': prefix_constructor('her ') - }, - 'his': { - 'en': prefix_constructor('his ') - }, - 'are_word': { - 'en': prefix_constructor('are ') - }, - 'is_word': { - 'en': prefix_constructor('is ') - }, - 'their': { - 'en': prefix_constructor('their ') - }, - 'my_possessive': { - 'en': prefix_constructor('my ') - }, - 'our_possessive': { - 'en': prefix_constructor('our ') - }, - 'of_the': { - 'en': prefix_constructor('of the ') - }, - 'your': { - 'en': prefix_constructor('your ') - }, - 'your_plural': { - 'en': prefix_constructor('your ') - }, - 'some': { - 'en': prefix_constructor('some ') - }, - 'its': { - 'en': prefix_constructor('its ') - }, - 'the': { - 'en': prefix_constructor('the ') - }, - 'these': { - 'en': prefix_constructor('these ') - }, - 'this': { - 'en': prefix_constructor('this ') - }, - 'does_a_b': { - 'en': prefix_constructor_two_arguments('does ') - }, - 'do_a_b': { - 'en': prefix_constructor_two_arguments('do ') - }, - 'did_a_b': { - 'en': prefix_constructor_two_arguments('did ') - }, - 'did_a_b_plural': { - 'en': prefix_constructor_two_arguments('did ') - }, - 'do_i': { - 'en': prefix_constructor('do I ') - }, - 'do_we': { - 'en': prefix_constructor('do we ') - }, - 'do_you': { - 'en': prefix_constructor('do you ') - }, - 'do_you_plural': { - 'en': prefix_constructor('do you ') - }, - 'did_i': { - 'en': prefix_constructor('did I ') - }, - 'did_we': { - 'en': prefix_constructor('did we ') - }, - 'did_you': { - 'en': prefix_constructor('did you ') - }, - 'did_you_plural': { - 'en': prefix_constructor('did you ') - }, - 'was_i': { - 'en': prefix_constructor('was I ') - }, - 'were_we': { - 'en': prefix_constructor('were we ') - }, - 'were_you': { - 'en': prefix_constructor('were you ') - }, - 'were_you_plural': { - 'en': prefix_constructor('were you ') - }, - 'was_a_b': { - 'en': prefix_constructor_two_arguments('was ') - }, - 'were_a_b': { - 'en': prefix_constructor_two_arguments('were ') - }, - 'were_a_b_plural': { - 'en': prefix_constructor_two_arguments('were ') - }, - 'have_i': { - 'en': prefix_constructor('have I ') - }, - 'have_we': { - 'en': prefix_constructor('have we ') - }, - 'have_you': { - 'en': prefix_constructor('have you ') - }, - 'have_you_plural': { - 'en': prefix_constructor('have you ') - }, - 'has_a_b': { - 'en': prefix_constructor_two_arguments('has ') - }, - 'have_a_b': { - 'en': prefix_constructor_two_arguments('have ') - }, - 'verb_past': { - 'en': verb_past_en, - 'es': verb_past_es, - 'de': verb_past_de, - 'fr': verb_past_fr, - 'it': verb_past_it, - 'nl': verb_past_nl - }, - 'verb_present': { - 'en': verb_present_en, - 'es': verb_present_es, - 'de': verb_present_de, - 'fr': verb_present_fr, - 'it': verb_present_it, - 'nl': verb_present_nl - }, - 'noun_plural': { - 'en': noun_plural_en, - 'es': noun_plural_es, - 'de': noun_plural_de, - 'fr': noun_plural_fr, - 'it': noun_plural_it, - 'nl': noun_plural_nl - }, - 'noun_singular': { - 'en': noun_singular_en, - 'es': noun_singular_es, - 'de': noun_singular_de, - 'fr': noun_singular_fr, - 'it': noun_singular_it, - 'nl': noun_singular_nl - }, - 'indefinite_article': { - 'en': indefinite_article_en, - 'es': indefinite_article_es, - 'de': indefinite_article_de, - 'it': indefinite_article_it - }, - 'currency_symbol': { - '*': currency_symbol_default - }, - 'period_list': { - '*': lambda: [[12, word("Per Month")], [1, word("Per Year")], [52, word("Per Week")], [24, word("Twice Per Month")], [26, word("Every Two Weeks")]] - }, - 'name_suffix': { - '*': lambda: ['Jr', 'Sr', 'II', 'III', 'IV', 'V', 'VI'] - }, - 'currency': { - '*': currency_default - }, - 'possessify': { - 'en': possessify_en - }, - 'possessify_long': { - 'en': double_prefix_constructor_reverse('the ', ' of the ') - }, - 'comma_and_list': { - 'en': comma_and_list_en, - 'es': comma_and_list_es, - 'de': comma_and_list_de - }, - 'comma_list': { - 'en': comma_list_en - }, - 'add_separators': { - 'en': add_separators_en - }, - 'nice_number': { - '*': nice_number_default - }, - 'quantity_noun': { - '*': quantity_noun_default - }, - 'ordinal_number': { - '*': ordinal_number_default - }, - 'ordinal': { - '*': ordinal_default - }, - 'capitalize': { - '*': capitalize_default - }, - 'title_case': { - '*': titlecasestr - }, - 'salutation': { - '*': salutation_default - } -} - - -def language_function_constructor(term): - - def func(*args, **kwargs): - ensure_definition(*args, **kwargs) - language = kwargs.get('language', None) - if language is None: - language = this_thread.language - if language in language_functions[term]: - return language_functions[term][language](*args, **kwargs) - if '*' in language_functions[term]: - return language_functions[term]['*'](*args, **kwargs) - if 'en' in language_functions[term]: - logmessage("Term " + str(term) + " is not defined for language " + str(language)) - return language_functions[term]['en'](*args, **kwargs) - raise SystemError("term " + str(term) + " not defined in language_functions for English or *") - return func - -in_the = language_function_constructor('in_the') -a_preposition_b = language_function_constructor('a_preposition_b') -a_in_the_b = language_function_constructor('a_in_the_b') -i_subjective = language_function_constructor('i_subjective') -he_subjective = language_function_constructor('he_subjective') -she_subjective = language_function_constructor('she_subjective') -genderless_subjective = language_function_constructor('genderless_subjective') -myself = language_function_constructor('myself') -itself = language_function_constructor('itself') -herself = language_function_constructor('herself') -himself = language_function_constructor('himself') -themselves = language_function_constructor('themselves') -genderless_self = language_function_constructor('genderless_self') -yourself = language_function_constructor('yourself') -yourselves = language_function_constructor('yourselves') -ourselves = language_function_constructor('ourselves') -you_subjective = language_function_constructor('you_subjective') -you_subjective_plural = language_function_constructor('you_subjective_plural') -we_subjective = language_function_constructor('we_subjective') -they_subjective = language_function_constructor('they_subjective') -it_subjective = language_function_constructor('it_subjective') -it_objective = language_function_constructor('it_objective') -them_objective = language_function_constructor('them_objective') -genderless_objective = language_function_constructor('genderless_objective') -me_objective = language_function_constructor('me_objective') -him_objective = language_function_constructor('him_objective') -her_objective = language_function_constructor('her_objective') -you_objective = language_function_constructor('you_objective') -you_objective_plural = language_function_constructor('you_objective_plural') -us_objective = language_function_constructor('us_objective') -are_we = language_function_constructor('are_we') -are_you = language_function_constructor('are_you') -are_you_plural = language_function_constructor('are_you_plural') -am_i = language_function_constructor('am_i') -her = language_function_constructor('her') -his = language_function_constructor('his') -are_word = language_function_constructor('are_word') -is_word = language_function_constructor('is_word') -their = language_function_constructor('their') -my_possessive = language_function_constructor('my_possessive') -our_possessive = language_function_constructor('our_possessive') -of_the = language_function_constructor('of_the') -your = language_function_constructor('your') -your_plural = language_function_constructor('your_plural') -some = language_function_constructor('some') -its = language_function_constructor('its') -the = language_function_constructor('the') -these = language_function_constructor('these') -this = language_function_constructor('this') -does_a_b = language_function_constructor('does_a_b') -do_a_b = language_function_constructor('do_a_b') -did_a_b = language_function_constructor('did_a_b') -did_a_b_plural = language_function_constructor('did_a_b_plural') -do_i = language_function_constructor('do_i') -do_we = language_function_constructor('do_we') -do_you = language_function_constructor('do_you') -do_you_plural = language_function_constructor('do_you_plural') -did_i = language_function_constructor('did_i') -did_we = language_function_constructor('did_we') -did_you = language_function_constructor('did_you') -did_you_plural = language_function_constructor('did_you_plural') -was_i = language_function_constructor('was_i') -were_we = language_function_constructor('were_we') -were_you = language_function_constructor('were_you') -were_you_plural = language_function_constructor('were_you_plural') -was_a_b = language_function_constructor('was_a_b') -were_a_b = language_function_constructor('were_a_b') -were_a_b_plural = language_function_constructor('were_a_b_plural') -have_i = language_function_constructor('have_i') -have_we = language_function_constructor('have_we') -have_you = language_function_constructor('have_you') -have_you_plural = language_function_constructor('have_you_plural') -has_a_b = language_function_constructor('has_a_b') -have_a_b = language_function_constructor('have_a_b') -verb_past = language_function_constructor('verb_past') -verb_present = language_function_constructor('verb_present') -noun_plural = language_function_constructor('noun_plural') -noun_singular = language_function_constructor('noun_singular') -indefinite_article = language_function_constructor('indefinite_article') -period_list = language_function_constructor('period_list') -name_suffix = language_function_constructor('name_suffix') -currency = language_function_constructor('currency') -currency_symbol = language_function_constructor('currency_symbol') -possessify = language_function_constructor('possessify') -possessify_long = language_function_constructor('possessify_long') -comma_list = language_function_constructor('comma_list') -comma_and_list = language_function_constructor('comma_and_list') -add_separators = language_function_constructor('add_separators') -nice_number = language_function_constructor('nice_number') -quantity_noun = language_function_constructor('quantity_noun') -capitalize = language_function_constructor('capitalize') -capitalize_function = capitalize -title_case = language_function_constructor('title_case') -ordinal_number = language_function_constructor('ordinal_number') -ordinal = language_function_constructor('ordinal') -salutation = language_function_constructor('salutation') - -if verb_past.__doc__ is None: - verb_past.__doc__ = """Return the past tense of a verb. - - Args: - verb (str): The verb to conjugate. - **kwargs: Optional conjugation parameters passed to the underlying - language function (e.g. ``'3gp'`` for third-person past tense). - - Returns: - str: The past-tense form of the verb (e.g. ``verb_past('help')`` - returns ``'helped'``). - """ -if verb_present.__doc__ is None: - verb_present.__doc__ = """Return the present tense of a verb. - - Args: - verb (str): The verb to conjugate (may be in any tense). - **kwargs: Optional conjugation parameters passed to the underlying - language function (e.g. ``'3sg'`` for third-person singular). - - Returns: - str: The present-tense form of the verb (e.g. - ``verb_present('helped', '3sg')`` returns ``'helps'``). - """ -if noun_plural.__doc__ is None: - noun_plural.__doc__ = """Return the plural form of a noun. - - Args: - noun (str): The noun to pluralize. - *pargs: An optional quantity (number, list, dict, or set). When the - quantity is exactly ``1`` the singular form is returned instead. - **kwargs: Pass ``noun_is_singular=True`` to skip singularization - before pluralizing. - - Returns: - str: The plural form of the noun, or the singular form if the - optional quantity equals ``1``. - """ -if noun_singular.__doc__ is None: - noun_singular.__doc__ = """Return the singular form of a noun. - - Args: - noun (str): The noun to singularize. - *pargs: An optional quantity (number, list, dict, or set). When the - quantity is not ``1`` the original noun is returned unchanged. - - Returns: - str: The singular form of the noun, or the original noun when the - optional quantity is not ``1``. - """ -if indefinite_article.__doc__ is None: - indefinite_article.__doc__ = """Return a noun preceded by the appropriate indefinite article. - - Args: - noun (str): The noun phrase to precede with an article. - **kwargs: Additional keyword arguments passed to the underlying - language function. - - Returns: - str: The noun prefixed with ``'a'`` or ``'an'`` as appropriate - (e.g. ``indefinite_article('apple')`` returns ``'an apple'``). - """ -if capitalize.__doc__ is None: - capitalize.__doc__ = """Return the input string with the first letter capitalized. - - Args: - a (str): The string to capitalize. - **kwargs: Additional keyword arguments passed to the underlying - language function. - - Returns: - str: The input string with its first character converted to - upper case. - """ -if period_list.__doc__ is None: - period_list.__doc__ = """Return a list of per-year period options for use in multiple-choice fields. - - Returns: - list: A list of ``[number, label]`` pairs representing common - payment periods (e.g. ``[[12, 'Per Month'], [1, 'Per Year'], - [52, 'Per Week'], ...]``). - """ -if name_suffix.__doc__ is None: - name_suffix.__doc__ = """Return a list of common name suffixes for use in multiple-choice fields. - - Returns: - list: A list of name suffix strings such as - ``['Jr', 'Sr', 'II', 'III', 'IV', 'V', 'VI']``. - """ -if currency.__doc__ is None: - currency.__doc__ = """Format a number as a currency value using the current locale. - - Args: - value: The numeric value to format. - **kwargs: Optional keyword arguments including ``decimals`` (bool, - default ``True``), ``symbol`` (str override for the currency - symbol), and ``symbol_precedes`` (bool controlling symbol - position). - Returns: - str: The formatted currency string (e.g. ``currency(45.2)`` returns - ``'$45.20'`` for a US locale). - """ -if currency_symbol.__doc__ is None: - currency_symbol.__doc__ = """Return the currency symbol for the current locale. - Returns: - str: The currency symbol (e.g. ``'$'`` for a US locale). Respects - overrides set via :func:`set_locale` or the ``currency symbol`` - configuration setting. - """ -if possessify.__doc__ is None: - possessify.__doc__ = """Return the possessive phrase combining two arguments. - Args: - a: The possessor. - b: The thing possessed. - **kwargs: Additional keyword arguments passed to the underlying - language function. +# def set_da_config(config): +# global daconfig +# daconfig = config - Returns: - str: A possessive phrase such as ``"a's b"``. - """ -if possessify_long.__doc__ is None: - possessify_long.__doc__ = """Return the long possessive phrase combining two arguments. + +def get_config(key, none_value=None): + """Return a value from the docassemble configuration file. Args: - a: The possessor. - b: The thing possessed. - **kwargs: Additional keyword arguments passed to the underlying - language function. + key (str): The configuration directive to look up. + none_value (optional): The value to return if the key is not found in + the configuration. Defaults to None. Returns: - str: A possessive phrase of the form ``"the b of a"``. + The configuration value associated with the key, or ``none_value`` + if the key is not present. """ -if comma_list.__doc__ is None: - comma_list.__doc__ = """Return the items joined by commas. + return get_configuration().get(key, none_value) - Args: - *pargs: Items to join, or a single iterable as the first argument. - **kwargs: Optional ``comma_string`` (default ``', '``) to customize - the separator. +# def set_default_language(lang): +# global default_language +# default_language = lang - Returns: - str: The items separated by commas (e.g. - ``comma_list('lions', 'tigers', 'bears')`` returns - ``'lions, tigers, bears'``). - """ -if comma_and_list.__doc__ is None: - comma_and_list.__doc__ = """Return the items joined by commas with "and" before the last item. +# def set_default_dialect(dialect): +# global default_dialect +# default_dialect = dialect +# return - Args: - *pargs: Items to join, or a single iterable as the first argument. - **kwargs: Optional keyword arguments including ``oxford`` (bool, - default ``True``), ``and_string`` (default ``'and'``), - ``comma_string``, ``before_and``, and ``after_and``. +# def set_default_country(country): +# global default_country +# default_country = country +# return - Returns: - str: An English-language listing such as ``'lions, tigers, and - bears'``. - """ -if add_separators.__doc__ is None: - add_separators.__doc__ = """Return the list items as strings with separators appended. +# def set_default_timezone(timezone): +# global default_timezone +# default_timezone = timezone +# return - Appends ``;`` to all items except the penultimate, which gets - ``'; and'``, and the last, which gets ``'.'``. - Args: - the_list: The list of items to process. - separator (str, optional): Separator appended to middle items. - Defaults to ``';'``. - last_separator (str, optional): Separator appended to the - penultimate item. Defaults to ``'; and'``. - end_mark (str, optional): Mark appended to the final item. - Defaults to ``'.'``. +# def reset_thread_local(): +# this_thread.open_files = set() +# this_thread.temporary_resources = set() - Returns: - list: A list of strings with separators appended. - """ -if nice_number.__doc__ is None: - nice_number.__doc__ = """Return a number expressed as a word for small values, or as a formatted numeral. +# def reset_thread_variables(): +# this_thread.saved_files = {} +# this_thread.message_log = [] - Args: - num: The number to convert. - **kwargs: Optional keyword arguments including ``capitalize`` - (bool), ``language`` (str), and ``use_word`` (bool, default - ``False``). - Returns: - str: The number as a word (e.g. ``nice_number(4)`` returns - ``'four'``) or as a locale-formatted numeral for larger values. - """ -if quantity_noun.__doc__ is None: - quantity_noun.__doc__ = """Return a number combined with a noun in the appropriate singular or plural form. +# def reset_local_variables(): +# # logmessage("reset_local_variables") +# this_thread.language = server.default_language +# this_thread.dialect = server.default_dialect +# this_thread.voice = server.default_voice +# this_thread.country = server.default_country +# this_thread.locale = server.default_locale +# this_thread.session_id = None +# this_thread.interview = None +# this_thread.interview_status = None +# this_thread.evaluation_context = None +# this_thread.gathering_mode = {} +# this_thread.global_vars = GenericObject() +# this_thread.current_variable = [] +# # this_thread.template_vars = [] +# this_thread.open_files = set() +# this_thread.saved_files = {} +# this_thread.message_log = [] +# this_thread.misc = {} +# this_thread.probing = False +# this_thread.current_info = {} +# this_thread.current_package = None +# this_thread.current_question = None +# this_thread.current_section = None +# this_thread.internal = {} +# this_thread.markdown = markdown.Markdown(extensions=['smarty', 'markdown.extensions.sane_lists', 'markdown.extensions.tables', 'markdown.extensions.attr_list', 'markdown.extensions.md_in_html', 'footnotes'], output_format='html5') +# this_thread.prevent_going_back = False - Combines :func:`nice_number` and :func:`noun_plural`. Rounds the number - to the nearest integer unless ``as_integer=False`` is passed. - Args: - num: The quantity. - noun (str): The singular noun. - **kwargs: Optional keyword arguments including ``as_integer`` - (bool, default ``True``) and other arguments accepted by - :func:`nice_number`. +def prevent_going_back(): + """Disable the back button so the user cannot revisit previous questions. - Returns: - str: The quantity and noun combined (e.g. ``quantity_noun(2, - 'apple')`` returns ``'two apples'``). + Once called, the user will not be able to go back and change any answers + entered before this point in the interview. """ -if title_case.__doc__ is None: - title_case.__doc__ = """Return the input string with the first letter of each word capitalized. + this_thread.prevent_going_back = True - Args: - a (str): The string to convert to title case. - **kwargs: Additional keyword arguments passed to the underlying - language function. - Returns: - str: The title-cased string (e.g. ``title_case('the importance of - being ernest')`` returns ``'The Importance of Being Ernest'``). - """ -if ordinal_number.__doc__ is None: - ordinal_number.__doc__ = """Return the ordinal form of a cardinal number. +def manual_line_breaks(text): + """Replaces newlines with manual line breaks.""" + if this_thread.evaluation_context == 'docx': + return re.sub(r' *\r?\n *', '', str(text)) + return re.sub(r' *\r?\n *', ' [BR] ', str(text)) - Args: - num: The cardinal number (1-based). - **kwargs: Optional keyword arguments including ``capitalize`` - (bool) and ``use_word`` (bool, default depends on the value). - Returns: - str: The ordinal form (e.g. ``ordinal_number(8)`` returns - ``'eighth'``; ``ordinal_number(8, use_word=False)`` returns - ``'8th'``). - """ -if ordinal.__doc__ is None: - ordinal.__doc__ = """Return the ordinal form of a zero-based index. +def need(*pargs): + """Ensure that the given variables are defined, asking questions if necessary. - Equivalent to ``ordinal_number(num + 1)``. This is useful when working - with zero-based list indexes. + Evaluating each argument causes docassemble to seek its definition + through the normal interview logic. The function always returns + ``True``. Using ``need()`` is purely for readability; writing + ``need(x, y)`` is equivalent to writing ``x; y`` in a code block. Args: - num: The zero-based index. - **kwargs: Optional keyword arguments passed to :func:`ordinal_number`. + *pargs: Variables whose definitions should be ensured. Returns: - str: The ordinal form (e.g. ``ordinal(0)`` returns ``'first'``; - ``ordinal(22)`` returns ``'23rd'``). + bool: Always ``True``. """ + ensure_definition(*pargs) + for argument in pargs: + argument # pylint: disable=pointless-statement + return True + + +def pickleable_objects(input_dict): + output_dict = {} + for key in input_dict: + if isinstance(input_dict[key], (types.ModuleType, types.FunctionType, TypeType, types.BuiltinFunctionType, types.BuiltinMethodType, types.MethodType, FileType)): + continue + if key == "__builtins__": + continue + output_dict[key] = input_dict[key] + return output_dict def underscore_to_space(a): @@ -4929,10 +3142,10 @@ def store_variables_snapshot(data=None, include_internal=False, key=None, persis if key is not None and not isinstance(key, str): raise DAError("store_variables_snapshot: key must be a string") if data is None: - the_data = serializable_dict(get_user_dict(), include_internal=include_internal) + the_data = serializable_dict(get_current_user_dict(), include_internal=include_internal) else: the_data = safe_json(data) - server.write_answer_json(session, filename, the_data, tags=key, persistent=bool(persistent)) + write_answer_json(session, filename, the_data, tags=key, persistent=bool(persistent)) def all_variables(simplify=True, include_internal=False, special=False, make_copy=False): @@ -4960,18 +3173,18 @@ def all_variables(simplify=True, include_internal=False, special=False, make_cop when ``special='tags'``. """ if special == 'titles': - return this_thread.interview.get_title(get_user_dict(), adapted=True) + return this_thread.interview.get_title(get_current_user_dict(), adapted=True) if special == 'metadata': return copy.deepcopy(this_thread.interview.consolidated_metadata) if special == 'tags': session_tags() return copy.deepcopy(this_thread.internal['tags']) if simplify: - return serializable_dict(get_user_dict(), include_internal=include_internal) + return serializable_dict(get_current_user_dict(), include_internal=include_internal) if make_copy: - new_dict = copy.deepcopy(pickleable_objects(get_user_dict())) + new_dict = copy.deepcopy(pickleable_objects(get_current_user_dict())) else: - new_dict = pickleable_objects(get_user_dict()) + new_dict = pickleable_objects(get_current_user_dict()) if not include_internal and '_internal' in new_dict: new_dict = copy.copy(new_dict) del new_dict['_internal'] @@ -5031,7 +3244,7 @@ def force_ask(*pargs, **kwargs): for item in the_pargs: if isinstance(item, str) and illegal_variable_name(item): raise DAError("Illegal variable name") - raise ForcedNameError(*the_pargs, user_dict=get_user_dict(), evaluate=kwargs.get('evaluate', False)) + raise ForcedNameError(*the_pargs, user_dict=get_current_user_dict(), evaluate=kwargs.get('evaluate', False)) force_ask_nameerror(the_pargs[0]) @@ -5062,7 +3275,7 @@ def force_gather(*pargs, forget_prior=False, evaluate=False): unique_id = this_thread.current_info['user']['session_uid'] if 'event_stack' in this_thread.internal and unique_id in this_thread.internal['event_stack']: this_thread.internal['event_stack'][unique_id] = [] - the_user_dict = get_user_dict() + the_user_dict = get_current_user_dict() the_context = {} for var_name in ('x', 'i', 'j', 'k', 'l', 'm', 'n'): if var_name in the_user_dict: @@ -5088,7 +3301,7 @@ def static_filename_path(filereference, return_nonexistent=False): else: result = package_data_filename(static_filename(filereference), return_nonexistent=return_nonexistent) # if result is None or not os.path.isfile(result): - # result = server.absolute_filename("/playgroundstatic/" + re.sub(r'[^A-Za-z0-9\-\_\. ]', '', filereference)).path + # result = absolute_filename("/playgroundstatic/" + re.sub(r'[^A-Za-z0-9\-\_\. ]', '', filereference)).path return result @@ -5189,7 +3402,7 @@ def package_template_filename(the_file, **kwargs): m = re.search(r'^docassemble\.playground([0-9]+)([A-Za-z]?[A-Za-z0-9]*)$', parts[0]) if m: parts[1] = re.sub(r'^data/templates/', '', parts[1]) - abs_file = server.absolute_filename("/playgroundtemplate/" + m.group(1) + '/' + (m.group(2) or 'default') + '/' + re.sub(r'[^A-Za-z0-9\-\_\. ]', '', parts[1])) # pylint: disable=assignment-from-none + abs_file = absolute_filename("/playgroundtemplate/" + m.group(1) + '/' + (m.group(2) or 'default') + '/' + re.sub(r'[^A-Za-z0-9\-\_\. ]', '', parts[1])) # pylint: disable=assignment-from-none if abs_file is None: return None return abs_file.path @@ -5236,12 +3449,12 @@ def package_data_filename(the_file, return_nonexistent=False): if m: if re.search(r'^data/sources/', parts[1]): parts[1] = re.sub(r'^data/sources/', '', parts[1]) - abs_file = server.absolute_filename("/playgroundsources/" + m.group(1) + '/' + (m.group(2) or 'default') + '/' + re.sub(r'[^A-Za-z0-9\-\_\. ]', '', parts[1])) # pylint: disable=assignment-from-none + abs_file = absolute_filename("/playgroundsources/" + m.group(1) + '/' + (m.group(2) or 'default') + '/' + re.sub(r'[^A-Za-z0-9\-\_\. ]', '', parts[1])) # pylint: disable=assignment-from-none if abs_file is None: return None return abs_file.path parts[1] = re.sub(r'^data/static/', '', parts[1]) - abs_file = server.absolute_filename("/playgroundstatic/" + m.group(1) + '/' + (m.group(2) or 'default') + '/' + re.sub(r'[^A-Za-z0-9\-\_\. ]', '', parts[1])) # pylint: disable=assignment-from-none + abs_file = absolute_filename("/playgroundstatic/" + m.group(1) + '/' + (m.group(2) or 'default') + '/' + re.sub(r'[^A-Za-z0-9\-\_\. ]', '', parts[1])) # pylint: disable=assignment-from-none if abs_file is None: return None return abs_file.path @@ -5256,7 +3469,7 @@ def package_data_filename(the_file, return_nonexistent=False): else: result = None # if result is None or not os.path.isfile(result): - # result = server.absolute_filename("/playgroundstatic/" + re.sub(r'[^A-Za-z0-9\-\_\.]', '', the_file)).path + # result = absolute_filename("/playgroundstatic/" + re.sub(r'[^A-Za-z0-9\-\_\.]', '', the_file)).path return result @@ -5336,7 +3549,7 @@ def process_action(): else: # logmessage("process_action: doing a gather of " + variable_name) if len(variable_dict['context']) > 0: - the_user_dict = get_user_dict() + the_user_dict = get_current_user_dict() for var_name, var_val in variable_dict['context'].items(): the_user_dict[var_name] = var_val del the_user_dict @@ -5351,7 +3564,7 @@ def process_action(): this_thread.current_info.update(event_info) the_context = event_info.get('context', {}) if len(the_context) > 0: - the_user_dict = get_user_dict() + the_user_dict = get_current_user_dict() for var_name, var_val in the_context.items(): the_user_dict[var_name] = var_val del the_user_dict @@ -5406,7 +3619,7 @@ def process_action(): for variable_name in this_thread.current_info['arguments']['variables']: if variable_name not in [(variable_dict if isinstance(variable_dict, str) else variable_dict['var']) for variable_dict in this_thread.internal['gather']]: the_context = {} - the_user_dict = get_user_dict() + the_user_dict = get_current_user_dict() for var_name in ('x', 'i', 'j', 'k', 'l', 'm', 'n'): if var_name in the_user_dict: the_context[var_name] = the_user_dict[var_name] @@ -5616,7 +3829,7 @@ def process_action(): for var in this_thread.current_info['arguments'][key]: if var not in [(variable_dict if isinstance(variable_dict, str) else variable_dict['var']) for variable_dict in this_thread.internal['gather']]: the_context = {} - the_user_dict = get_user_dict() + the_user_dict = get_current_user_dict() for var_name in ('x', 'i', 'j', 'k', 'l', 'm', 'n'): if var_name in the_user_dict: the_context[var_name] = the_user_dict[var_name] @@ -5624,7 +3837,7 @@ def process_action(): this_thread.internal['gather'].append({'var': var, 'context': the_context}) elif this_thread.current_info['arguments'][key] not in [(variable_dict if isinstance(variable_dict, str) else variable_dict['var']) for variable_dict in this_thread.internal['gather']]: the_context = {} - the_user_dict = get_user_dict() + the_user_dict = get_current_user_dict() for var_name in ('x', 'i', 'j', 'k', 'l', 'm', 'n'): if var_name in the_user_dict: the_context[var_name] = the_user_dict[var_name] @@ -5689,7 +3902,7 @@ def myb64unquote(text): def debug_status(): - return server.debug + return get_debug_status() # grep -E -R -o -h "word\(['\"][^\)]+\)" * | sed "s/^[^'\"]+['\"]//g" @@ -5743,20 +3956,20 @@ def repad_byte(text): return text + (equals_byte * ((4 - len(text) % 4) % 4)) -class lister(ast.NodeVisitor): +class Lister(ast.NodeVisitor): def __init__(self): self.stack = [] - def visit_Name(self, node): + def visit_Name(self, node): # pylint: disable=invalid-name self.stack.append(['name', node.id]) ast.NodeVisitor.generic_visit(self, node) - def visit_Attribute(self, node): + def visit_Attribute(self, node): # pylint: disable=invalid-name self.stack.append(['attr', node.attr]) ast.NodeVisitor.generic_visit(self, node) - def visit_Subscript(self, node): + def visit_Subscript(self, node): # pylint: disable=invalid-name self.stack.append(['index', re.sub(r'\n', '', astunparse.unparse(node.slice))]) ast.NodeVisitor.generic_visit(self, node) # def visit_BinOp(self, node): @@ -5768,7 +3981,7 @@ def visit_Subscript(self, node): def components_of(full_variable): node = ast.parse(full_variable, mode='eval') - crawler = lister() + crawler = Lister() crawler.visit(node) components = list(reversed(crawler.stack)) start_index = 0 @@ -5778,18 +3991,8 @@ def components_of(full_variable): return components[start_index:] -def get_user_dict(): - frame = sys._getframe(1) - while frame is not None: - f_locals = frame.f_locals - if 'user_dict' in f_locals: - user_dict = f_locals['user_dict'] - if isinstance(user_dict, dict) and '_internal' in user_dict: - return user_dict - if '_internal' in f_locals: - return f_locals - frame = frame.f_back - return {} +get_user_dict = get_current_user_dict + def invalidate(*pargs): """Make one or more variables undefined while remembering their prior values as defaults. @@ -5820,18 +4023,7 @@ def _undefine_internal_old(*pargs, invalidate=False): # pylint: disable=redefin raise DAError("undefine: variable " + repr(var) + " is not a valid variable name") if len(vars_to_delete) == 0: return - frame = sys._getframe(1) - the_user_dict = frame.f_locals - while '_internal' not in the_user_dict: - frame = frame.f_back - if frame is None: - return - if 'user_dict' in frame.f_locals: - the_user_dict = frame.f_locals['user_dict'] - if '_internal' in the_user_dict: - break - return - the_user_dict = frame.f_locals + the_user_dict = get_current_user_dict() this_thread.probing = True if invalidate: for var in vars_to_delete: @@ -5847,58 +4039,59 @@ def _undefine_internal_old(*pargs, invalidate=False): # pylint: disable=redefin this_thread.probing = False -def _undefine_internal_new(*pargs, invalidate=False): # pylint: disable=redefined-outer-name - vars_to_delete = [] - the_pargs = unpack_pargs(pargs) - for var in the_pargs: - str(var) - if not isinstance(var, str): - raise DAError("undefine() must be given a string, not " + repr(var) + ", a " + str(var.__class__.__name__)) - try: - eval(var, {}) - continue - except: - vars_to_delete.append(var) - components = components_of(var) - if len(components) == 0 or len(components[0]) < 2: - raise DAError("undefine: variable " + repr(var) + " is not a valid variable name") - if len(vars_to_delete) == 0: - return - frame = sys._getframe(1) - the_user_dict = frame.f_locals - the_user_dict_g = frame.f_globals - while '_internal' not in the_user_dict: - frame = frame.f_back - if frame is None: - return - if 'user_dict' in frame.f_locals: - the_user_dict = frame.f_locals['user_dict'] - the_user_dict_g = frame.f_globals - if '_internal' in the_user_dict: - break - return - the_user_dict = frame.f_locals - the_user_dict_g = frame.f_globals - this_thread.probing = True - if invalidate: - for var in vars_to_delete: - try: - exec("_internal['dirty'][" + repr(var) + "] = " + var, the_user_dict_g, the_user_dict) - except: - pass - for var in vars_to_delete: - try: - exec('del ' + var, the_user_dict_g, the_user_dict) - except: - pass - this_thread.probing = False - - -if python313: - _undefine_internal = _undefine_internal_new -else: - _undefine_internal = _undefine_internal_old - +# def _undefine_internal_new(*pargs, invalidate=False): # pylint: disable=redefined-outer-name +# vars_to_delete = [] +# the_pargs = unpack_pargs(pargs) +# for var in the_pargs: +# str(var) +# if not isinstance(var, str): +# raise DAError("undefine() must be given a string, not " + repr(var) + ", a " + str(var.__class__.__name__)) +# try: +# eval(var, {}) +# continue +# except: +# vars_to_delete.append(var) +# components = components_of(var) +# if len(components) == 0 or len(components[0]) < 2: +# raise DAError("undefine: variable " + repr(var) + " is not a valid variable name") +# if len(vars_to_delete) == 0: +# return +# frame = sys._getframe(1) +# the_user_dict = frame.f_locals +# the_user_dict_g = frame.f_globals +# while '_internal' not in the_user_dict: +# frame = frame.f_back +# if frame is None: +# return +# if 'user_dict' in frame.f_locals: +# the_user_dict = frame.f_locals['user_dict'] +# the_user_dict_g = frame.f_globals +# if '_internal' in the_user_dict: +# break +# return +# the_user_dict = frame.f_locals +# the_user_dict_g = frame.f_globals +# this_thread.probing = True +# if invalidate: +# for var in vars_to_delete: +# try: +# exec("_internal['dirty'][" + repr(var) + "] = " + var, the_user_dict_g, the_user_dict) +# except: +# pass +# for var in vars_to_delete: +# try: +# exec('del ' + var, the_user_dict_g, the_user_dict) +# except: +# pass +# this_thread.probing = False + + +# if python313: +# _undefine_internal = _undefine_internal_new +# else: +# _undefine_internal = _undefine_internal_old + +_undefine_internal = _undefine_internal_old def undefine(*pargs, invalidate=False): # pylint: disable=redefined-outer-name """Delete one or more interview variables, making them undefined. @@ -5960,11 +4153,11 @@ def set_variables(variables, process_objects=False): variables = variables.elements if not isinstance(variables, dict): raise DAError("set_variables: argument must be a dictionary") - user_dict = get_user_dict() + user_dict = get_current_user_dict() if user_dict is None: raise DAError("set_variables: could not find interview answers") if process_objects: - variables = server.transform_json_variables(variables) # pylint: disable=assignment-from-none + variables = transform_json_variables(variables) # pylint: disable=assignment-from-none for var, val in variables.items(): exec(var + " = None", user_dict) user_dict['__define_val'] = val @@ -5987,16 +4180,15 @@ def define(var, val): ensure_definition(var, val) if not isinstance(var, str) or not re.search(r'^[A-Za-z_]', var): raise DAError("define() must be given a string as the variable name") - user_dict = get_user_dict() + user_dict = get_current_user_dict() if user_dict is None: raise DAError("define: could not find interview answers") - # Trigger exceptions for the left hand side before creating __define_val - exec(var + " = None", user_dict) - # logmessage("Got past the lhs check") user_dict['__define_val'] = val - exec(var + " = __define_val", user_dict) - if '__define_val' in user_dict: - del user_dict['__define_val'] + try: + exec(var + " = __define_val", user_dict) + finally: + if '__define_val' in user_dict: + del user_dict['__define_val'] class DefCaller(Enum): @@ -6014,15 +4206,15 @@ def is_predicate(self) -> bool: return self == self.DEFINED -def _defined_internal_with_prior(var, caller: DefCaller, alt=None): +def _inspect_user_dict_with_prior(var, caller: DefCaller, alt=None): try: - return _defined_internal(var, caller, alt=alt, prior=True) + return _inspect_user_dict(var, caller, alt=alt, prior=True) except: - return _defined_internal(var, caller, alt=alt) + return _inspect_user_dict(var, caller, alt=alt) -def _defined_internal_old(var, caller: DefCaller, alt=None, prior=False): - """Checks if a variable is defined at all in the stack. Used by defined(), +def _inspect_user_dict(var, caller: DefCaller, alt=None, prior=False): + """Checks if a variable is defined. Used by defined(), value(), and showifdef(). `var` is the name of the variable to check, `caller` is the name of the function calling (which determines what to do if the variable is found to be defined or not). @@ -6033,30 +4225,13 @@ def _defined_internal_old(var, caller: DefCaller, alt=None, prior=False): user all of the questions necessary to answer it * SHOWIFDEF, then the value if returned, but only if no questions have to be asked """ - frame = sys._getframe(1) components = components_of(var) if len(components) == 0 or len(components[0]) < 2: raise DAError("defined: variable " + repr(var) + " is not a valid variable name") variable = components[0][1] - the_user_dict = frame.f_locals + the_user_dict = get_old_user_dict() if prior else get_current_user_dict() failure_val = False if caller.is_predicate() else alt - user_dict_name = 'old_user_dict' if prior else 'user_dict' - while (variable not in the_user_dict) or prior: - frame = frame.f_back - if frame is None: - if caller.is_pure(): - return failure_val - force_ask_nameerror(variable) - if user_dict_name in frame.f_locals: - the_user_dict = frame.f_locals[user_dict_name] - if variable in the_user_dict: - break - if caller.is_pure(): - return failure_val - force_ask_nameerror(variable) - else: - the_user_dict = frame.f_locals - if variable not in the_user_dict: + if the_user_dict is None or variable not in the_user_dict: if caller.is_pure(): return failure_val force_ask_nameerror(variable) @@ -6149,142 +4324,141 @@ def _defined_internal_old(var, caller: DefCaller, alt=None, prior=False): return eval(cum_variable, the_user_dict) -def _defined_internal_new(var, caller: DefCaller, alt=None, prior=False): - """Checks if a variable is defined at all in the stack. Used by defined(), - value(), and showifdef(). `var` is the name of the variable to check, - `caller` is the name of the function calling (which determines what to do - if the variable is found to be defined or not). - - if caller is: - * DEFINED, then True/False is returned depending on if the variable is defined - * VALUE, then the actual value of the variable is returned, after asking the - user all of the questions necessary to answer it - * SHOWIFDEF, then the value if returned, but only if no questions have to be asked - """ - frame = sys._getframe(1) - components = components_of(var) - if len(components) == 0 or len(components[0]) < 2: - raise DAError("defined: variable " + repr(var) + " is not a valid variable name") - variable = components[0][1] - the_user_dict = frame.f_locals - the_user_dict_g = frame.f_globals - failure_val = False if caller.is_predicate() else alt - user_dict_name = 'old_user_dict' if prior else 'user_dict' - while (variable not in the_user_dict) or prior: - frame = frame.f_back - if frame is None: - if caller.is_pure(): - return failure_val - force_ask_nameerror(variable) - if user_dict_name in frame.f_locals: - the_user_dict = frame.f_locals[user_dict_name] - the_user_dict_g = frame.f_globals - if variable in the_user_dict: - break - if caller.is_pure(): - return failure_val - force_ask_nameerror(variable) - else: - the_user_dict = frame.f_locals - the_user_dict_g = frame.f_globals - if variable not in the_user_dict: - if caller.is_pure(): - return failure_val - force_ask_nameerror(variable) - if len(components) == 1: - if caller.is_predicate(): - return True - return eval(variable, the_user_dict_g, the_user_dict) - cum_variable = '' - if caller.is_pure(): - this_thread.probing = True - has_random_instance_name = False - for elem in components: - if elem[0] == 'name': - cum_variable = elem[1] - continue - if elem[0] == 'attr': - base_var = cum_variable - to_eval = "hasattr(" + cum_variable + ", " + repr(elem[1]) + ")" - cum_variable += '.' + elem[1] - try: - result = eval(to_eval, the_user_dict_g, the_user_dict) - except: - if caller.is_pure(): - this_thread.probing = False - return failure_val - force_ask_nameerror(base_var) - if result: - continue - if caller.is_pure(): - this_thread.probing = False - return failure_val - the_cum = eval(base_var, the_user_dict_g, the_user_dict) - try: - if not the_cum.has_nonrandom_instance_name: - has_random_instance_name = True - except: - pass - if has_random_instance_name: - force_ask_nameerror(cum_variable) - getattr(the_cum, elem[1]) - elif elem[0] == 'index': - try: - the_index = eval(elem[1], the_user_dict_g, the_user_dict) - except: - if caller.is_pure(): - this_thread.probing = False - return failure_val - value(elem[1]) - try: - the_cum = eval(cum_variable, the_user_dict_g, the_user_dict) - except: - if caller.is_pure(): - this_thread.probing = False - return failure_val - force_ask_nameerror(cum_variable) - if hasattr(the_cum, 'instanceName') and hasattr(the_cum, 'elements'): - var_elements = cum_variable + '.elements' - else: - var_elements = cum_variable - if isinstance(the_index, int): - to_eval = 'len(' + var_elements + ') > ' + str(the_index) - else: - to_eval = elem[1] + " in " + var_elements - cum_variable += '[' + elem[1] + ']' - try: - result = eval(to_eval, the_user_dict_g, the_user_dict) - except: - # the evaluation probably will never fail because we know the base variable is defined - if caller.is_pure(): - this_thread.probing = False - return failure_val - force_ask_nameerror(cum_variable) - if result: - continue - if caller.is_pure(): - this_thread.probing = False - return failure_val - try: - if not the_cum.has_nonrandom_instance_name: - has_random_instance_name = True - except: - pass - if has_random_instance_name: - force_ask_nameerror(cum_variable) - the_cum[the_index] # pylint: disable=pointless-statement - if caller.is_pure(): - this_thread.probing = False - if caller.is_predicate(): - return True - return eval(cum_variable, the_user_dict_g, the_user_dict) - - -if python313: - _defined_internal = _defined_internal_new -else: - _defined_internal = _defined_internal_old +# def _inspect_user_dict_new(var, caller: DefCaller, alt=None, prior=False): +# """Checks if a variable is defined at all in the stack. Used by defined(), +# value(), and showifdef(). `var` is the name of the variable to check, +# `caller` is the name of the function calling (which determines what to do +# if the variable is found to be defined or not). +# if caller is: +# * DEFINED, then True/False is returned depending on if the variable is defined +# * VALUE, then the actual value of the variable is returned, after asking the +# user all of the questions necessary to answer it +# * SHOWIFDEF, then the value if returned, but only if no questions have to be asked +# """ +# frame = sys._getframe(1) +# components = components_of(var) +# if len(components) == 0 or len(components[0]) < 2: +# raise DAError("defined: variable " + repr(var) + " is not a valid variable name") +# variable = components[0][1] +# the_user_dict = frame.f_locals +# the_user_dict_g = frame.f_globals +# failure_val = False if caller.is_predicate() else alt +# user_dict_name = 'old_user_dict' if prior else 'user_dict' +# while (variable not in the_user_dict) or prior: +# frame = frame.f_back +# if frame is None: +# if caller.is_pure(): +# return failure_val +# force_ask_nameerror(variable) +# if user_dict_name in frame.f_locals: +# the_user_dict = frame.f_locals[user_dict_name] +# the_user_dict_g = frame.f_globals +# if variable in the_user_dict: +# break +# if caller.is_pure(): +# return failure_val +# force_ask_nameerror(variable) +# else: +# the_user_dict = frame.f_locals +# the_user_dict_g = frame.f_globals +# if variable not in the_user_dict: +# if caller.is_pure(): +# return failure_val +# force_ask_nameerror(variable) +# if len(components) == 1: +# if caller.is_predicate(): +# return True +# return eval(variable, the_user_dict_g, the_user_dict) +# cum_variable = '' +# if caller.is_pure(): +# this_thread.probing = True +# has_random_instance_name = False +# for elem in components: +# if elem[0] == 'name': +# cum_variable = elem[1] +# continue +# if elem[0] == 'attr': +# base_var = cum_variable +# to_eval = "hasattr(" + cum_variable + ", " + repr(elem[1]) + ")" +# cum_variable += '.' + elem[1] +# try: +# result = eval(to_eval, the_user_dict_g, the_user_dict) +# except: +# if caller.is_pure(): +# this_thread.probing = False +# return failure_val +# force_ask_nameerror(base_var) +# if result: +# continue +# if caller.is_pure(): +# this_thread.probing = False +# return failure_val +# the_cum = eval(base_var, the_user_dict_g, the_user_dict) +# try: +# if not the_cum.has_nonrandom_instance_name: +# has_random_instance_name = True +# except: +# pass +# if has_random_instance_name: +# force_ask_nameerror(cum_variable) +# getattr(the_cum, elem[1]) +# elif elem[0] == 'index': +# try: +# the_index = eval(elem[1], the_user_dict_g, the_user_dict) +# except: +# if caller.is_pure(): +# this_thread.probing = False +# return failure_val +# value(elem[1]) +# try: +# the_cum = eval(cum_variable, the_user_dict_g, the_user_dict) +# except: +# if caller.is_pure(): +# this_thread.probing = False +# return failure_val +# force_ask_nameerror(cum_variable) +# if hasattr(the_cum, 'instanceName') and hasattr(the_cum, 'elements'): +# var_elements = cum_variable + '.elements' +# else: +# var_elements = cum_variable +# if isinstance(the_index, int): +# to_eval = 'len(' + var_elements + ') > ' + str(the_index) +# else: +# to_eval = elem[1] + " in " + var_elements +# cum_variable += '[' + elem[1] + ']' +# try: +# result = eval(to_eval, the_user_dict_g, the_user_dict) +# except: +# # the evaluation probably will never fail because we know the base variable is defined +# if caller.is_pure(): +# this_thread.probing = False +# return failure_val +# force_ask_nameerror(cum_variable) +# if result: +# continue +# if caller.is_pure(): +# this_thread.probing = False +# return failure_val +# try: +# if not the_cum.has_nonrandom_instance_name: +# has_random_instance_name = True +# except: +# pass +# if has_random_instance_name: +# force_ask_nameerror(cum_variable) +# the_cum[the_index] # pylint: disable=pointless-statement +# if caller.is_pure(): +# this_thread.probing = False +# if caller.is_predicate(): +# return True +# return eval(cum_variable, the_user_dict_g, the_user_dict) + + +# if python313: +# _inspect_user_dict = _inspect_user_dict_new +# else: +# _inspect_user_dict = _inspect_user_dict_old def value(var: str, prior=False): """Return the value of an interview variable specified by name. @@ -6312,8 +4486,8 @@ def value(var: str, prior=False): if re.search(r'[\(\)\n\r]|lambda:|lambda ', var): raise DAError("value() is invalid: " + repr(var)) if prior: - return _defined_internal_with_prior(var, DefCaller.VALUE) - return _defined_internal(var, DefCaller.VALUE) + return _inspect_user_dict_with_prior(var, DefCaller.VALUE) + return _inspect_user_dict(var, DefCaller.VALUE) def defined(var: str, prior=False) -> bool: @@ -6342,8 +4516,8 @@ def defined(var: str, prior=False) -> bool: except: pass if prior: - return _defined_internal_with_prior(var, DefCaller.VALUE) - return _defined_internal(var, DefCaller.DEFINED) + return _inspect_user_dict_with_prior(var, DefCaller.VALUE) + return _inspect_user_dict(var, DefCaller.DEFINED) def showifdef(var: str, alternative='', prior=False): @@ -6374,8 +4548,8 @@ def showifdef(var: str, alternative='', prior=False): if re.search(r'[\(\)\n\r]|lambda:|lambda ', var): raise DAError("showifdef() is invalid: " + repr(var)) if prior: - return _defined_internal_with_prior(var, DefCaller.SHOWIFDEF, alt=alternative) - return _defined_internal(var, DefCaller.SHOWIFDEF, alt=alternative, prior=prior) + return _inspect_user_dict_with_prior(var, DefCaller.SHOWIFDEF, alt=alternative) + return _inspect_user_dict(var, DefCaller.SHOWIFDEF, alt=alternative, prior=prior) def illegal_variable_name(var): @@ -6385,7 +4559,7 @@ def illegal_variable_name(var): t = ast.parse(var) except: return True - detector = docassemble.base.astparser.detectIllegal() + detector = docassemble.base.astparser.DetectIllegal() detector.visit(t) return detector.illegal @@ -6697,12 +4871,12 @@ def referring_url(default=None, current=False): URL. """ if current: - url = server.get_referer() + url = get_referer() else: url = this_thread.internal.get('referer', None) if url is None: if default is None: - default = server.daconfig.get('exitpage', 'https://docassemble.org') + default = get_configuration().get('exitpage', 'https://docassemble.org') url = default return url @@ -6790,11 +4964,6 @@ def italic(text, default=None): return '_' + str(default) + '_' return '_' + re.sub(r'\_', '', str(text)) + '_' -# def inspector(): -# frame = inspect.stack()[1][0] -# for key in frame.__dict__.keys(): -# logmessage(str(key)) - def indent(text, by=None): """Indent each line of the text by a number of spaces. @@ -7064,11 +5233,11 @@ def interview_list(exclude_invalid=True, action=None, filename=None, session=Non raise DAError("interview_list: invalid next_id.") else: start_id = None - (the_list, start_id) = server.user_interviews(user_id=user_id, secret=this_thread.current_info['secret'], exclude_invalid=exclude_invalid, action=action, filename=filename, session=session, include_dict=include_dict, delete_shared=delete_shared, start_id=start_id, query=query) # pylint: disable=assignment-from-none,unpacking-non-sequence + (the_list, start_id) = user_interviews(user_id=user_id, secret=this_thread.current_info['secret'], exclude_invalid=exclude_invalid, action=action, filename=filename, session=session, include_dict=include_dict, delete_shared=delete_shared, start_id=start_id, query=query) # pylint: disable=assignment-from-none,unpacking-non-sequence if start_id is None: return (the_list, None) return (the_list, myb64quote(str(start_id))) - return server.user_interviews(user_id=user_id, secret=this_thread.current_info['secret'], exclude_invalid=exclude_invalid, action=action, filename=filename, session=session, include_dict=include_dict, delete_shared=delete_shared, query=query) + return user_interviews(user_id=user_id, secret=this_thread.current_info['secret'], exclude_invalid=exclude_invalid, action=action, filename=filename, session=session, include_dict=include_dict, delete_shared=delete_shared, query=query) return None @@ -7080,7 +5249,7 @@ def interview_menu(*pargs, **kwargs): keys such as ``title``, ``filename``, ``link``, ``tags``, and ``metadata``. """ - return server.interview_menu(*pargs, **kwargs) + return server_interview_menu(*pargs, **kwargs) def get_user_list(include_inactive=False, next_id=None): @@ -7106,7 +5275,7 @@ def get_user_list(include_inactive=False, next_id=None): raise DAError("get_user_list: invalid next_id.") else: start_id = None - (the_list, start_id) = server.get_user_list(include_inactive=include_inactive, start_id=start_id) # pylint: disable=assignment-from-none,unpacking-non-sequence + (the_list, start_id) = server_get_user_list(include_inactive=include_inactive, start_id=start_id) # pylint: disable=assignment-from-none,unpacking-non-sequence if start_id is None: return (the_list, None) return (the_list, myb64quote(str(start_id))) @@ -7137,19 +5306,19 @@ def manage_privileges(*pargs): else: the_command = arglist.pop(0) if the_command == 'list': - return server.get_privileges_list() + return get_privileges_list() if the_command == 'inspect': if len(arglist) != 1: raise DAError("manage_privileges: invalid number of arguments") - return server.get_permissions_of_privilege(arglist[0]) + return get_permissions_of_privilege(arglist[0]) if the_command == 'add': for priv in arglist: - server.add_privilege(priv) + add_privilege(priv) if len(arglist) > 0: return True elif the_command == 'remove': for priv in arglist: - server.remove_privilege(priv) + remove_privilege(priv) if len(arglist) > 0: return True else: @@ -7175,7 +5344,7 @@ def get_user_info(user_id=None, email=None): """ if this_thread.current_info['user']['is_authenticated'] and user_id is None and email is None: user_id = this_thread.current_info['user']['the_user_id'] - return server.get_user_info(user_id=user_id, email=email) + return server_get_user_info(user_id=user_id, email=email) def set_user_info(**kwargs): @@ -7193,7 +5362,7 @@ def set_user_info(**kwargs): """ user_id = kwargs.get('user_id', None) email = kwargs.get('email', None) - server.set_user_info(**kwargs) + server_set_user_info(**kwargs) if 'privileges' in kwargs and isinstance(kwargs['privileges'], (list, tuple)) and len(kwargs['privileges']) > 0: this_thread.current_info['user']['roles'] = list(kwargs['privileges']) if (user_id is None and email is None) or (user_id is not None and user_id == this_thread.current_info['user']['theid']) or (email is not None and email == this_thread.current_info['user']['email']): @@ -7222,7 +5391,7 @@ def create_user(email, password, privileges=None, info=None): Returns: int: The user ID of the newly created account. """ - return server.create_user(email, password, privileges=privileges, info=info) + return server_create_user(email, password, privileges=privileges, info=info) def invite_user(email_address, privilege=None, send=True): @@ -7243,7 +5412,7 @@ def invite_user(email_address, privilege=None, send=True): str or None: The registration URL when ``send=False``, otherwise ``None``. """ - return server.invite_user(email_address, privilege=privilege, send=send) + return server_invite_user(email_address, privilege=privilege, send=send) def get_user_secret(username, password): @@ -7261,7 +5430,7 @@ def get_user_secret(username, password): str or None: The decryption key string if the credentials are valid, otherwise ``None``. """ - return server.get_secret(username, password) + return server_get_secret(username, password) def create_session(yaml_filename, secret=None, url_args=None): @@ -7280,7 +5449,7 @@ def create_session(yaml_filename, secret=None, url_args=None): """ if secret is None: secret = this_thread.current_info.get('secret', None) - (encrypted, session_id) = server.create_session(yaml_filename, secret, url_args=url_args) # pylint: disable=assignment-from-none,unpacking-non-sequence + (encrypted, session_id) = server_create_session(yaml_filename, secret, url_args=url_args) # pylint: disable=assignment-from-none,unpacking-non-sequence if secret is None and encrypted: raise DAError("create_session: the interview is encrypted but you did not provide a secret.") return session_id @@ -7306,7 +5475,7 @@ def get_session_variables(yaml_filename, session_id, secret=None, simplify=True) raise DAError("You cannot get variables from the current interview session") if secret is None: secret = this_thread.current_info.get('secret', None) - return server.get_session_variables(yaml_filename, session_id, secret=secret, simplify=simplify) + return server_get_session_variables(yaml_filename, session_id, secret=secret, simplify=simplify) def set_session_variables(yaml_filename, session_id, variables, secret=None, question_name=None, overwrite=False, process_objects=False, delete=None): @@ -7339,7 +5508,7 @@ def set_session_variables(yaml_filename, session_id, variables, secret=None, que delete = [delete] else: delete = list(delete) - server.set_session_variables(yaml_filename, session_id, variables, secret=secret, del_variables=delete, question_name=question_name, post_setting=not overwrite, process_objects=process_objects) + server_set_session_variables(yaml_filename, session_id, variables, secret=secret, del_variables=delete, question_name=question_name, post_setting=not overwrite, process_objects=process_objects) def run_action_in_session(yaml_filename, session_id, action, arguments=None, secret=None, persistent=False, overwrite=False, read_only=False): @@ -7372,7 +5541,7 @@ def run_action_in_session(yaml_filename, session_id, action, arguments=None, sec arguments = {} if secret is None: secret = this_thread.current_info.get('secret', None) - result = server.run_action_in_session(i=yaml_filename, session=session_id, secret=secret, action=action, persistent=persistent, overwrite=overwrite, read_only=read_only, arguments=arguments) + result = server_run_action_in_session(i=yaml_filename, session=session_id, secret=secret, action=action, persistent=persistent, overwrite=overwrite, read_only=read_only, arguments=arguments) if isinstance(result, dict): if result['status'] == 'success': return True @@ -7398,7 +5567,7 @@ def get_question_data(yaml_filename, session_id, secret=None): raise DAError("You cannot get question data from the current interview session") if secret is None: secret = this_thread.current_info.get('secret', None) - return server.get_question_data(yaml_filename, session_id, secret) + return server_get_question_data(yaml_filename, session_id, secret) def go_back_in_session(yaml_filename, session_id, secret=None): @@ -7416,7 +5585,7 @@ def go_back_in_session(yaml_filename, session_id, secret=None): raise DAError("You cannot go back in the current interview session") if secret is None: secret = this_thread.current_info.get('secret', None) - server.go_back_in_session(yaml_filename, session_id, secret=secret) + server_go_back_in_session(yaml_filename, session_id, secret=secret) def turn_to_at_sign(match): @@ -7489,15 +5658,6 @@ def redact(text): return output -def ensure_definition(*pargs, **kwargs): - for val in pargs: - if isinstance(val, Undefined): - str(val) - for val in kwargs.values(): - if isinstance(val, Undefined): - str(val) - - def verbatim(text): """Return the text with special formatting characters escaped for the current output context. @@ -7580,7 +5740,7 @@ def re_run_logic(): def intrinsic_name_of(var_name, the_user_dict=None): if the_user_dict is None: - the_user_dict = get_user_dict() + the_user_dict = get_current_user_dict() from docassemble.base.util import DAObject # pylint: disable=import-outside-toplevel expression_as_list = [x for x in match_brackets_or_dot.split(var_name) if x != ''] n = len(expression_as_list) @@ -7600,7 +5760,7 @@ def intrinsic_name_of(var_name, the_user_dict=None): def intrinsic_names_of(*pargs, the_user_dict=None): if the_user_dict is None: - the_user_dict = get_user_dict() + the_user_dict = get_current_user_dict() output = [] for parg in pargs: if isinstance(parg, str): @@ -7762,10 +5922,15 @@ def empty(cls): class ServerContext: - pass + """Class with one attribute, context, which indicates whether the web server or the websockets server is running""" + def __init__(self, context): + self.context = context + + def set_context(self, context): + self.context = context + -server_context = ServerContext() -server_context.context = 'web' +server_context = ServerContext('web') def get_action_stack(): diff --git a/docassemble_base/docassemble/base/geocode.py b/docassemble_base/docassemble/base/geocode.py index 3a0f8225c..084260174 100644 --- a/docassemble_base/docassemble/base/geocode.py +++ b/docassemble_base/docassemble/base/geocode.py @@ -1,13 +1,11 @@ -from docassemble.base.logger import logmessage from geopy.geocoders import GoogleV3 from geopy.geocoders import AzureMaps +from docassemble.base.logger import logmessage +from docassemble.base.config import daconfig class GeoCoder: - def __init__(self, *pargs, **kwargs): # pylint: disable=unused-argument - self.server = kwargs['server'] - def geocode(self, *pargs, **kwargs): self.data = self.geocoder.geocode(*pargs, **kwargs) return True @@ -17,14 +15,14 @@ class GoogleV3GeoCoder(GeoCoder): def config_ok(self): try: - assert isinstance(self.server.daconfig['google']['api key'], str) + assert isinstance(daconfig['google']['api key'], str) except: logmessage("geocode: cannot geocode without an 'api key' under 'google' in the Configuration. Set 'geolocate service' in the Configuration to use a different geocoding service.") return False return True def initialize(self): - self.geocoder = GoogleV3(api_key=self.server.daconfig['google']['api key']) + self.geocoder = GoogleV3(api_key=daconfig['google']['api key']) def populate_address(self, address): if 'formatted_address' in self.data.raw: @@ -145,14 +143,14 @@ class AzureMapsGeoCoder(GeoCoder): def config_ok(self): try: - assert isinstance(self.server.daconfig['azure maps']['primary key'], str) + assert isinstance(daconfig['azure maps']['primary key'], str) except: logmessage("geocode: cannot geocode without a 'primary key' under 'azure maps' in the Configuration. Set 'geolocate service' in the Configuration to use a different geocoding service.") return False return True def initialize(self): - self.geocoder = AzureMaps(self.server.daconfig['azure maps']['primary key']) + self.geocoder = AzureMaps(daconfig['azure maps']['primary key']) def populate_address(self, address): if 'address' not in self.data.raw: diff --git a/docassemble_base/docassemble/base/helpers.py b/docassemble_base/docassemble/base/helpers.py new file mode 100644 index 000000000..dc9d6379d --- /dev/null +++ b/docassemble_base/docassemble/base/helpers.py @@ -0,0 +1,29 @@ +import re + +nameerror_match = re.compile(r'\'(.*)\' (is not defined|referenced before assignment|is undefined|where it is not)') + +def extract_missing_name(the_error): + # logmessage("extract_missing_name: string was " + str(string)) + m = nameerror_match.search(str(the_error)) + if m: + return m.group(1) + raise the_error + + +def fix_quotes(match): + instring = match.group(1) + n = len(instring) + output = '' + i = 0 + while i < n: + if instring[i] == '\u201c' or instring[i] == '\u201d': + output += '"' + elif instring[i] == '\u2018' or instring[i] == '\u2019': + output += "'" + elif instring[i] == '&' and i + 4 < n and instring[i:i+5] == '&': + output += '&' + i += 4 + else: + output += instring[i] + i += 1 + return output diff --git a/docassemble_base/docassemble/base/hooks.py b/docassemble_base/docassemble/base/hooks.py new file mode 100644 index 000000000..16f95475c --- /dev/null +++ b/docassemble_base/docassemble/base/hooks.py @@ -0,0 +1,553 @@ +# pylint: disable=unused-argument +from typing import Any +from .plugin_manager import pm + + +def get_default_language() -> str: + return pm.hook.get_default_language() + +def get_default_dialect() -> str: + return pm.hook.get_default_dialect() + +def get_default_locale() -> str: + return pm.hook.get_default_locale() + +def get_default_voice() -> str: + return pm.hook.get_default_voice() + +def get_default_timezone() -> str: + return pm.hook.get_default_timezone() + +def get_default_country() -> str: + return pm.hook.get_default_country() + +def get_configuration() -> dict: + return pm.hook.get_configuration() + +def get_hostname() -> str: + return pm.hook.get_hostname() + +def get_debug_status() -> bool: + return pm.hook.get_debug_status() + +def save_numbered_file(filename, orig_path, yaml_file_name=None, uid=None) -> tuple: + return pm.hook.save_numbered_file( + filename=filename, + orig_path=orig_path, + yaml_file_name=yaml_file_name, + uid=uid, + ) + +def send_mail(the_message, config='default') -> None: + return pm.hook.send_mail(the_message=the_message, config=config) + +def absolute_filename(the_file) -> Any: + return pm.hook.absolute_filename(the_file=the_file) + +def write_record(key, data) -> int: + return pm.hook.write_record(key=key, data=data) + +def read_records(key) -> Any: + return pm.hook.read_records(key=key) + +def delete_record(key, the_id) -> Any: + return pm.hook.delete_record(key=key, the_id=the_id) + +def generate_csrf(secret_key=None, token_key=None) -> Any: + return pm.hook.generate_csrf(secret_key=secret_key, token_key=token_key) + +def url_for(endpoint, **kwargs) -> Any: + return pm.hook.url_for(endpoint=endpoint, kwargs=kwargs) + +def get_new_file_number(user_code, file_name, yaml_file_name=None) -> Any: + return pm.hook.get_new_file_number( + user_code=user_code, + file_name=file_name, + yaml_file_name=yaml_file_name, + ) + +def get_ext_and_mimetype(filename) -> Any: + return pm.hook.get_ext_and_mimetype(filename=filename) + +def file_finder(file_reference, question=None, folder=None, package=None, filename=None, return_nonexistent=False, uids=None) -> Any: + return pm.hook.file_finder( + file_reference=file_reference, + question=question, + folder=folder, + package=package, + filename=filename, + return_nonexistent=return_nonexistent, + uids=uids, + ) + +def file_number_finder(file_number, filename=None, uids=None, privileged=False) -> Any: + return pm.hook.file_number_finder( + file_number=file_number, + filename=filename, + uids=uids, + privileged=privileged, + ) + +def server_sql_get(key, secret=None) -> Any: + return pm.hook.server_sql_get(key=key, secret=secret) + +def server_sql_defined(key) -> Any: + return pm.hook.server_sql_defined(key=key) + +def server_sql_set(key, val, encrypted=True, secret=None, the_user_id=None) -> Any: + return pm.hook.server_sql_set( + key=key, + val=val, + encrypted=encrypted, + secret=secret, + the_user_id=the_user_id, + ) + +def server_sql_delete(key) -> Any: + return pm.hook.server_sql_delete(key=key) + +def server_sql_keys(prefix) -> Any: + return pm.hook.server_sql_keys(prefix=prefix) + +def alchemy_url(db_config) -> Any: + return pm.hook.alchemy_url(db_config=db_config) + +def connect_args(db_config) -> Any: + return pm.hook.connect_args(db_config=db_config) + +def get_default_table_class() -> Any: + return pm.hook.get_default_table_class() + +def get_default_thead_class() -> Any: + return pm.hook.get_default_thead_class() + +def to_text(html_doc) -> Any: + return pm.hook.to_text(html_doc=html_doc) + +def url_finder(file_reference, **kwargs) -> Any: + return pm.hook.url_finder(file_reference=file_reference, kwargs=kwargs) + +def navigation_bar(nav, interview, wrapper=True, inner_div_class=None, inner_div_extra=None, show_links=None, hide_inactive_subs=True, a_class=None, show_nesting=True, include_arrows=False, always_open=False, return_dict=None) -> Any: + return pm.hook.navigation_bar( + nav=nav, + interview=interview, + wrapper=wrapper, + inner_div_class=inner_div_class, + inner_div_extra=inner_div_extra, + show_links=show_links, + hide_inactive_subs=hide_inactive_subs, + a_class=a_class, + show_nesting=show_nesting, + include_arrows=include_arrows, + always_open=always_open, + return_dict=return_dict, + ) + +def chat_partners_available(session_id, yaml_filename, the_user_id, mode, partner_roles) -> Any: + return pm.hook.chat_partners_available( + session_id=session_id, + yaml_filename=yaml_filename, + the_user_id=the_user_id, + mode=mode, + partner_roles=partner_roles, + ) + +def get_chat_log(yaml_filename, session_id, secret, utc=True, timezone=None) -> Any: + return pm.hook.get_chat_log( + yaml_filename=yaml_filename, + session_id=session_id, + secret=secret, + utc=utc, + timezone=timezone, + ) + +def sms_body(phone_number, body='question', config='default') -> Any: + return pm.hook.sms_body(phone_number=phone_number, body=body, config=config) + +def send_fax(fax_number, the_file, config, country=None) -> Any: + return pm.hook.send_fax( + fax_number=fax_number, + the_file=the_file, + config=config, + country=country, + ) + +def get_sms_session(phone_number, config='default') -> Any: + return pm.hook.get_sms_session(phone_number=phone_number, config=config) + +def initiate_sms_session(phone_number, yaml_filename=None, uid=None, secret=None, encrypted=None, user_id=None, email=None, new=False, config='default') -> Any: + return pm.hook.initiate_sms_session( + phone_number=phone_number, + yaml_filename=yaml_filename, + uid=uid, + secret=secret, + encrypted=encrypted, + user_id=user_id, + email=email, + new=new, + config=config, + ) + +def terminate_sms_session(phone_number, config='default') -> Any: + return pm.hook.terminate_sms_session(phone_number=phone_number, config=config) + +def applock(action, application, maxtime=4) -> Any: + return pm.hook.applock(action=action, application=application, maxtime=maxtime) + +def get_twilio_config() -> Any: + return pm.hook.get_twilio_config() + +def get_server_redis() -> Any: + return pm.hook.get_server_redis() + +def get_server_redis_user() -> Any: + return pm.hook.get_server_redis_user() + +def get_user_object(user_id) -> Any: + return pm.hook.get_user_object(user_id=user_id) + +def user_id_dict() -> Any: + return pm.hook.user_id_dict() + +def retrieve_email(email_id) -> Any: + return pm.hook.retrieve_email(email_id=email_id) + +def retrieve_emails(**kwargs) -> Any: + return pm.hook.retrieve_emails(kwargs=kwargs) + +def get_short_code(**kwargs) -> Any: + return pm.hook.get_short_code(kwargs) + +def make_png_for_pdf(doc, prefix, page=None) -> Any: + return pm.hook.make_png_for_pdf(doc=doc, prefix=prefix, page=page) + +def ocr_google_in_background(image_file, raw_result, user_code) -> Any: + return pm.hook.ocr_google_in_background( + image_file=image_file, + raw_result=raw_result, + user_code=user_code, + ) + +def task_ready(task_id) -> Any: + return pm.hook.task_ready(task_id=task_id) + +def wait_for_task(task_id, timeout=None) -> Any: + return pm.hook.wait_for_task(task_id=task_id, timeout=timeout) + +def user_interviews(user_id=None, secret=None, exclude_invalid=True, action=None, filename=None, session=None, tag=None, include_dict=True, delete_shared=False, admin=False, start_id=None, temp_user_id=None, query=None, minimal=False) -> Any: + return pm.hook.user_interviews( + user_id=user_id, + secret=secret, + exclude_invalid=exclude_invalid, + action=action, + filename=filename, + session=session, + tag=tag, + include_dict=include_dict, + delete_shared=delete_shared, + admin=admin, + start_id=start_id, + temp_user_id=temp_user_id, + query=query, + minimal=minimal, + ) + +def server_interview_menu(absolute_urls=False, start_new=False, tag=None) -> Any: + return pm.hook.server_interview_menu( + absolute_urls=absolute_urls, + start_new=start_new, + tag=tag, + ) + +def server_get_user_list(include_inactive=False, start_id=None) -> Any: + return pm.hook.server_get_user_list( + include_inactive=include_inactive, + start_id=start_id, + ) + +def server_get_user_info(user_id=None, email=None, case_sensitive=False, admin=False) -> Any: + return pm.hook.server_get_user_info( + user_id=user_id, + email=email, + case_sensitive=case_sensitive, + admin=admin, + ) + +def server_set_user_info(**kwargs) -> Any: + return pm.hook.server_set_user_info(kwargs=kwargs) + +def make_user_inactive(user_id=None, email=None) -> Any: + return pm.hook.make_user_inactive(user_id=user_id, email=email) + +def server_get_secret(username, password, case_sensitive=False) -> Any: + return pm.hook.server_get_secret( + username=username, + password=password, + case_sensitive=case_sensitive, + ) + +def server_get_session_variables(yaml_filename, session_id, secret=None, simplify=True, use_lock=False) -> Any: + return pm.hook.server_get_session_variables( + yaml_filename=yaml_filename, + session_id=session_id, + secret=secret, + simplify=simplify, + use_lock=use_lock, + ) + +def server_go_back_in_session(yaml_filename, session_id, secret=None, return_question=False, use_lock=False, encode=False) -> Any: + return pm.hook.server_go_back_in_session( + yaml_filename=yaml_filename, + session_id=session_id, + secret=secret, + return_question=return_question, + use_lock=use_lock, + encode=encode, + ) + +def server_create_session(yaml_filename, secret, url_args=None, referer=None, req=None) -> Any: + return pm.hook.server_create_session( + yaml_filename=yaml_filename, + secret=secret, + url_args=url_args, + referer=referer, + req=req, + ) + +def server_set_session_variables(yaml_filename, session_id, variables, secret=None, return_question=False, literal_variables=None, del_variables=None, question_name=None, event_list=None, advance_progress_meter=False, post_setting=True, use_lock=False, encode=False, process_objects=False) -> Any: + return pm.hook.server_set_session_variables( + yaml_filename=yaml_filename, + session_id=session_id, + variables=variables, + secret=secret, + return_question=return_question, + literal_variables=literal_variables, + del_variables=del_variables, + question_name=question_name, + event_list=event_list, + advance_progress_meter=advance_progress_meter, + post_setting=post_setting, + use_lock=use_lock, + encode=encode, + process_objects=process_objects, + ) + +def get_privileges_list(admin=False) -> Any: + return pm.hook.get_privileges_list(admin=admin) + +def add_privilege(privilege) -> Any: + return pm.hook.add_privilege(privilege=privilege) + +def remove_privilege(privilege) -> Any: + return pm.hook.remove_privilege(privilege=privilege) + +def add_user_privilege(user_id, privilege) -> Any: + return pm.hook.add_user_privilege(user_id=user_id, privilege=privilege) + +def remove_user_privilege(user_id, privilege) -> Any: + return pm.hook.remove_user_privilege(user_id=user_id, privilege=privilege) + +def get_permissions_of_privilege(privilege, privileged=False) -> Any: + return pm.hook.get_permissions_of_privilege(privilege=privilege, privileged=privileged) + +def server_create_user(email, password, privileges=None, info=None) -> Any: + return pm.hook.server_create_user( + email=email, + password=password, + privileges=privileges, + info=info, + ) + +def file_set_attributes(file_number, **kwargs) -> Any: + return pm.hook.file_set_attributes( + file_number=file_number, + private=kwargs.get('private', None), + persistent=kwargs.get('persistent', None), + session=kwargs.get('session', None), + filename=kwargs.get('filename', None), + ) + +def file_user_access(file_number, allow_user_id=None, allow_email=None, disallow_user_id=None, disallow_email=None, disallow_all=False) -> Any: + return pm.hook.file_user_access( + file_number=file_number, + allow_user_id=allow_user_id, + allow_email=allow_email, + disallow_user_id=disallow_user_id, + disallow_email=disallow_email, + disallow_all=disallow_all, + ) + +def file_privilege_access(file_number, allow=None, disallow=None, disallow_all=False) -> Any: + return pm.hook.file_privilege_access( + file_number=file_number, + allow=allow, + disallow=disallow, + disallow_all=disallow_all, + ) + +def fg_make_png_for_pdf(doc, prefix, page=None) -> Any: + return pm.hook.fg_make_png_for_pdf(doc=doc, prefix=prefix, page=page) + +def fg_make_png_for_pdf_path(path, prefix, page=None) -> Any: + return pm.hook.fg_make_png_for_pdf_path(path=path, prefix=prefix, page=page) + +def fg_make_pdf_for_word_path(path, extension) -> Any: + return pm.hook.fg_make_pdf_for_word_path(path=path, extension=extension) + +def server_get_question_data(yaml_filename, session_id, secret, use_lock=True, user_dict=None, steps=None, is_encrypted=None, old_user_dict=None, save=True, post_setting=False, advance_progress_meter=False, action=None, encode=False) -> Any: + return pm.hook.server_get_question_data( + yaml_filename=yaml_filename, + session_id=session_id, + secret=secret, + use_lock=use_lock, + user_dict=user_dict, + steps=steps, + is_encrypted=is_encrypted, + old_user_dict=old_user_dict, + save=save, + post_setting=post_setting, + advance_progress_meter=advance_progress_meter, + action=action, + encode=encode, + ) + +def fix_pickle_obj(data) -> Any: + return pm.hook.fix_pickle_obj(data=data) + +def get_main_page_parts() -> Any: + return pm.hook.get_main_page_parts() + +def get_saved_file_class() -> Any: + return pm.hook.get_saved_file_class() + +def path_from_reference(file_reference) -> Any: + return pm.hook.path_from_reference(file_reference=file_reference) + +def get_button_class_prefix() -> Any: + return pm.hook.get_button_class_prefix() + +def write_answer_json(user_code, filename, data, tags=None, persistent=False) -> Any: + return pm.hook.write_answer_json( + user_code=user_code, + filename=filename, + data=data, + tags=tags, + persistent=persistent, + ) + +def read_answer_json(user_code, filename, tags=None, all_tags=False) -> Any: + return pm.hook.read_answer_json( + user_code=user_code, + filename=filename, + tags=tags, + all_tags=all_tags, + ) + +def delete_answer_json(user_code, filename, tags=None, delete_all=False, delete_persistent=False) -> Any: + return pm.hook.delete_answer_json( + user_code=user_code, + filename=filename, + tags=tags, + delete_all=delete_all, + delete_persistent=delete_persistent, + ) + +def variables_snapshot_connection() -> Any: + return pm.hook.variables_snapshot_connection() + +def variables_snapshot_connect() -> Any: + return pm.hook.variables_snapshot_connect() + +def get_referer() -> Any: + return pm.hook.get_referer() + +def stash_data(data, expire) -> Any: + return pm.hook.stash_data(data=data, expire=expire) + +def retrieve_stashed_data(key, secret, delete=False, refresh=False) -> Any: + return pm.hook.retrieve_stashed_data(key=key, secret=secret, delete=delete, refresh=refresh) + +def secure_filename_spaces_ok(filename) -> Any: + return pm.hook.secure_filename_spaces_ok(filename=filename) + +def secure_filename_unicode_ok(the_filename) -> Any: + return pm.hook.secure_filename_unicode_ok(the_filename=the_filename) + +def secure_filename(filename) -> Any: + return pm.hook.secure_filename(filename=filename) + +def transform_json_variables(obj) -> Any: + return pm.hook.transform_json_variables(obj=obj) + +def get_login_url(**kwargs) -> Any: + return pm.hook.get_login_url(kwargs=kwargs) + +def server_run_action_in_session(**kwargs) -> Any: + return pm.hook.server_run_action_in_session(kwargs=kwargs) + +def server_invite_user(email_address, privilege=None, send=True) -> Any: + return pm.hook.server_invite_user( + email_address=email_address, + privilege=privilege, + send=send, + ) + +def get_url() -> Any: + return pm.hook.get_url() + +def release_lock(user_code, filename) -> Any: + return pm.hook.release_lock(user_code=user_code, filename=filename) + +def register_db(db_name) -> Any: + return pm.hook.register_db(db_name=db_name) + +def create_objects_in_db(db_name) -> Any: + return pm.hook.create_objects_in_db(db_name=db_name) + +def get_cloud() -> Any: + return pm.hook.get_cloud() + +def cloud_custom(provider, config) -> Any: + return pm.hook.cloud_custom(provider=provider, config=config) + +def google_api() -> Any: + return pm.hook.google_api() + +def get_mail_class() -> Any: + return pm.hook.get_mail_class() + +def get_celery_app() -> Any: + return pm.hook.get_celery_app() + +def get_task(obj) -> Any: + return pm.hook.get_task(obj=obj) + +def chord(arg) -> Any: + return pm.hook.chord(arg=arg) + +def fix_ml_files(playground_number, current_project) -> Any: + return pm.hook.fix_ml_files(playground_number=playground_number, current_project=current_project) + +def write_ml_source(playground, playground_number, current_project, filename, finalize=True): + return pm.hook.write_ml_source(playground=playground, playground_number=playground_number, current_project=current_project, filename=filename, finalize=finalize) + +def ensure_training_loaded(interview) -> Any: + return pm.hook.ensure_training_loaded(interview=interview) + +def manage_chat_logs(mode: int, **kwargs) -> None: + return pm.hook.manage_chat_logs(mode=mode, kwargs=kwargs) + +def manage_global_objects(mode: int, **kwargs) -> None: + return pm.hook.manage_global_objects(mode=mode, kwargs=kwargs) + +def manage_email_server_objects(mode: int, **kwargs) -> None: + return pm.hook.manage_email_server_objects(mode=mode, kwargs=kwargs) + +def manage_tts_objects(mode: int, **kwargs) -> None: + return pm.hook.manage_tts_objects(mode=mode, kwargs=kwargs) + +def get_chat_log_internal(chat_mode, yaml_filename, session_id, user_id, temp_user_id, secret, self_user_id, self_temp_id) -> Any: + return pm.hook.get_chat_log(chat_mode=chat_mode, yaml_filename=yaml_filename, session_id=session_id, user_id=user_id, temp_user_id=temp_user_id, secret=secret, self_user_id=self_user_id, self_temp_id=self_temp_id) + +def get_ml_info(varname, default_package, default_file) -> Any: + return pm.hook.get_ml_info(varname=varname, default_package=default_package, default_file=default_file) diff --git a/docassemble_base/docassemble/base/hookspecs.py b/docassemble_base/docassemble/base/hookspecs.py new file mode 100644 index 000000000..2eb834595 --- /dev/null +++ b/docassemble_base/docassemble/base/hookspecs.py @@ -0,0 +1,506 @@ +# mypy: disable-error-code="empty-body" +# pylint: disable=unused-argument + +from typing import Any +import pluggy + +hookspec = pluggy.HookspecMarker("docassemble") + +@hookspec(firstresult=True) +def get_default_language() -> str: + """Default language""" + +@hookspec(firstresult=True) +def get_default_dialect() -> str: + """Default dialect""" + +@hookspec(firstresult=True) +def get_default_locale() -> str: + """Default locale""" + +@hookspec(firstresult=True) +def get_default_voice() -> str: + """Default voice""" + +@hookspec(firstresult=True) +def get_default_timezone() -> str: + """Return the default timezone string for the server. + + Returns the server's local timezone unless a default timezone is configured + in the docassemble configuration. + + Returns: + str: A timezone string such as ``'America/New_York'``. + """ + +@hookspec(firstresult=True) +def get_default_country() -> str: + """Default country""" + +@hookspec(firstresult=True) +def get_configuration() -> dict: + """Get configuration""" + +@hookspec(firstresult=True) +def get_hostname() -> str: + """Get hostname""" + +@hookspec(firstresult=True) +def get_debug_status() -> bool: + """Get debug status""" + +@hookspec(firstresult=True) +def save_numbered_file(filename, orig_path, yaml_file_name, uid) -> tuple: + """Save numbered file""" + +@hookspec(firstresult=True) +def send_mail(the_message, config) -> None: + """Send email""" + +@hookspec(firstresult=True) +def absolute_filename(the_file) -> Any: + """Get SavedFile or None for a file path""" + +@hookspec(firstresult=True) +def write_record(key, data) -> int: + """Store data in the SQL database under the given key. + + Args: + key (str): A string key to associate with the record. + data: The data to store. Must be pickleable. + + Returns: + int: The unique integer ID of the saved record. + """ + +@hookspec(firstresult=True) +def read_records(key) -> Any: + """Return all records stored under the given key. + + Args: + key (str): The string key used when calling ``write_record()``. + + Returns: + dict: A dictionary mapping unique integer record IDs to the stored data. + """ + +@hookspec(firstresult=True) +def delete_record(key, the_id) -> Any: + """Delete a record from the SQL database by key and ID. + + Args: + key (str): The string key associated with the record. + the_id (int): The unique integer ID of the record to delete. + """ + +@hookspec(firstresult=True) +def generate_csrf(secret_key, token_key) -> Any: + """Generate CSRF token""" + +@hookspec(firstresult=True) +def url_for(endpoint, kwargs) -> Any: + """Wrapper for flask url_for function; kwargs is a dict of keyword arguments""" + +@hookspec(firstresult=True) +def get_new_file_number(user_code, file_name, yaml_file_name) -> Any: + """Returns file number for a file""" + +@hookspec(firstresult=True) +def get_ext_and_mimetype(filename) -> Any: + """Returns file extension and mimetype""" + +@hookspec(firstresult=True) +def file_finder(file_reference, question, folder, package, filename, return_nonexistent, uids) -> Any: + """General-purpose retriever of a file by its reference""" + +@hookspec(firstresult=True) +def file_number_finder(file_number, filename, uids, privileged) -> Any: + """Returns information about a file based on its number""" + +@hookspec(firstresult=True) +def server_sql_get(key, secret) -> Any: + pass + +@hookspec(firstresult=True) +def server_sql_defined(key) -> Any: + pass + +@hookspec(firstresult=True) +def server_sql_set(key, val, encrypted, secret, the_user_id) -> Any: + pass + +@hookspec(firstresult=True) +def server_sql_delete(key) -> Any: + pass + +@hookspec(firstresult=True) +def server_sql_keys(prefix) -> Any: + pass + +@hookspec(firstresult=True) +def alchemy_url(db_config) -> Any: + pass + +@hookspec(firstresult=True) +def connect_args(db_config) -> Any: + pass + +@hookspec(firstresult=True) +def get_default_table_class() -> Any: + pass + +@hookspec(firstresult=True) +def get_default_thead_class() -> Any: + pass + +@hookspec(firstresult=True) +def to_text(html_doc) -> Any: + pass + +@hookspec(firstresult=True) +def url_finder(file_reference, kwargs) -> Any: + """Find a URL for a file reference; kwargs is a dict of keyword arguments""" + +@hookspec(firstresult=True) +def navigation_bar(nav, interview, wrapper, inner_div_class, inner_div_extra, show_links, hide_inactive_subs, a_class, show_nesting, include_arrows, always_open, return_dict) -> Any: + pass + +@hookspec(firstresult=True) +def chat_partners_available(session_id, yaml_filename, the_user_id, mode, partner_roles) -> Any: + pass + +@hookspec(firstresult=True) +def get_chat_log(yaml_filename, session_id, secret, utc, timezone) -> Any: + pass + +@hookspec(firstresult=True) +def sms_body(phone_number, body, config) -> Any: + pass + +@hookspec(firstresult=True) +def send_fax(fax_number, the_file, config, country) -> Any: + pass + +@hookspec(firstresult=True) +def get_sms_session(phone_number, config) -> Any: + pass + +@hookspec(firstresult=True) +def initiate_sms_session(phone_number, yaml_filename, uid, secret, encrypted, user_id, email, new, config) -> Any: + pass + +@hookspec(firstresult=True) +def terminate_sms_session(phone_number, config) -> Any: + pass + +@hookspec(firstresult=True) +def applock(action, application, maxtime) -> Any: + pass + +@hookspec(firstresult=True) +def get_twilio_config() -> Any: + pass + +@hookspec(firstresult=True) +def get_server_redis() -> Any: + pass + +@hookspec(firstresult=True) +def get_server_redis_user() -> Any: + pass + +@hookspec(firstresult=True) +def get_user_object(user_id) -> Any: + pass + +@hookspec(firstresult=True) +def user_id_dict() -> Any: + pass + +@hookspec(firstresult=True) +def retrieve_email(email_id) -> Any: + pass + +@hookspec(firstresult=True) +def retrieve_emails(kwargs) -> Any: + """Retrieve emails; kwargs is a dict of keyword arguments""" + +@hookspec(firstresult=True) +def get_short_code(kwargs) -> Any: + """Get short code; kwargs is a dict of keyword arguments""" + +@hookspec(firstresult=True) +def make_png_for_pdf(doc, prefix, page) -> Any: + pass + +@hookspec(firstresult=True) +def ocr_google_in_background(image_file, raw_result, user_code) -> Any: + pass + +@hookspec(firstresult=True) +def task_ready(task_id) -> Any: + pass + +@hookspec(firstresult=True) +def wait_for_task(task_id, timeout) -> Any: + pass + +@hookspec(firstresult=True) +def user_interviews(user_id, secret, exclude_invalid, action, filename, session, tag, include_dict, delete_shared, admin, start_id, temp_user_id, query, minimal) -> Any: + pass + +@hookspec(firstresult=True) +def server_interview_menu(absolute_urls, start_new, tag) -> Any: + pass + +@hookspec(firstresult=True) +def server_get_user_list(include_inactive, start_id) -> Any: + pass + +@hookspec(firstresult=True) +def server_get_user_info(user_id, email, case_sensitive, admin) -> Any: + pass + +@hookspec(firstresult=True) +def server_set_user_info(kwargs) -> Any: + """Set user info; kwargs is a dict of keyword arguments""" + +@hookspec(firstresult=True) +def make_user_inactive(user_id, email) -> Any: + pass + +@hookspec(firstresult=True) +def server_get_secret(username, password, case_sensitive) -> Any: + pass + +@hookspec(firstresult=True) +def server_get_session_variables(yaml_filename, session_id, secret, simplify, use_lock) -> Any: + pass + +@hookspec(firstresult=True) +def server_go_back_in_session(yaml_filename, session_id, secret, return_question, use_lock, encode) -> Any: + pass + +@hookspec(firstresult=True) +def server_create_session(yaml_filename, secret, url_args, referer, req) -> Any: + pass + +@hookspec(firstresult=True) +def server_set_session_variables(yaml_filename, session_id, variables, secret, return_question, literal_variables, del_variables, question_name, event_list, advance_progress_meter, post_setting, use_lock, encode, process_objects) -> Any: + pass + +@hookspec(firstresult=True) +def get_privileges_list(admin) -> Any: + pass + +@hookspec(firstresult=True) +def add_privilege(privilege) -> Any: + pass + +@hookspec(firstresult=True) +def remove_privilege(privilege) -> Any: + pass + +@hookspec(firstresult=True) +def add_user_privilege(user_id, privilege) -> Any: + pass + +@hookspec(firstresult=True) +def remove_user_privilege(user_id, privilege) -> Any: + pass + +@hookspec(firstresult=True) +def get_permissions_of_privilege(privilege, privileged) -> Any: + pass + +@hookspec(firstresult=True) +def server_create_user(email, password, privileges, info) -> Any: + pass + +@hookspec(firstresult=True) +def file_set_attributes(file_number, private, persistent, session, filename) -> Any: + """Set attributes on a stored file""" + +@hookspec(firstresult=True) +def file_user_access(file_number, allow_user_id, allow_email, disallow_user_id, disallow_email, disallow_all) -> Any: + pass + +@hookspec(firstresult=True) +def file_privilege_access(file_number, allow, disallow, disallow_all) -> Any: + pass + +@hookspec(firstresult=True) +def fg_make_png_for_pdf(doc, prefix, page) -> Any: + pass + +@hookspec(firstresult=True) +def fg_make_png_for_pdf_path(path, prefix, page) -> Any: + pass + +@hookspec(firstresult=True) +def fg_make_pdf_for_word_path(path, extension) -> Any: + pass + +@hookspec(firstresult=True) +def server_get_question_data(yaml_filename, session_id, secret, use_lock, user_dict, steps, is_encrypted, old_user_dict, save, post_setting, advance_progress_meter, action, encode) -> Any: + pass + +@hookspec(firstresult=True) +def fix_pickle_obj(data) -> Any: + pass + +@hookspec(firstresult=True) +def get_main_page_parts() -> Any: + pass + +@hookspec(firstresult=True) +def get_saved_file_class() -> Any: + pass + +@hookspec(firstresult=True) +def path_from_reference(file_reference) -> Any: + pass + +@hookspec(firstresult=True) +def get_button_class_prefix() -> Any: + pass + +@hookspec(firstresult=True) +def write_answer_json(user_code, filename, data, tags, persistent) -> Any: + pass + +@hookspec(firstresult=True) +def read_answer_json(user_code, filename, tags, all_tags) -> Any: + pass + +@hookspec(firstresult=True) +def delete_answer_json(user_code, filename, tags, delete_all, delete_persistent) -> Any: + pass + +@hookspec(firstresult=True) +def variables_snapshot_connection() -> Any: + pass + +@hookspec(firstresult=True) +def variables_snapshot_connect() -> Any: + pass + +@hookspec(firstresult=True) +def get_referer() -> Any: + pass + +@hookspec(firstresult=True) +def stash_data(data, expire) -> Any: + pass + +@hookspec(firstresult=True) +def retrieve_stashed_data(key, secret, delete, refresh) -> Any: + pass + +@hookspec(firstresult=True) +def secure_filename_spaces_ok(filename) -> Any: + pass + +@hookspec(firstresult=True) +def secure_filename_unicode_ok(the_filename) -> Any: + pass + +@hookspec(firstresult=True) +def secure_filename(filename) -> Any: + pass + +@hookspec(firstresult=True) +def transform_json_variables(obj) -> Any: + pass + +@hookspec(firstresult=True) +def get_login_url(kwargs) -> Any: + """Get login URL; kwargs is a dict of keyword arguments""" + +@hookspec(firstresult=True) +def server_run_action_in_session(kwargs) -> Any: + """Run action in session; kwargs is a dict of keyword arguments""" + +@hookspec(firstresult=True) +def server_invite_user(email_address, privilege, send) -> Any: + pass + +@hookspec(firstresult=True) +def get_url() -> Any: + pass + +@hookspec(firstresult=True) +def release_lock(user_code, filename) -> Any: + pass + +@hookspec(firstresult=True) +def register_db(db_name) -> Any: + pass + +@hookspec(firstresult=True) +def create_objects_in_db(db_name) -> Any: + pass + +@hookspec(firstresult=True) +def get_cloud() -> Any: + pass + +@hookspec(firstresult=True) +def cloud_custom(provider, config) -> Any: + pass + +@hookspec(firstresult=True) +def google_api() -> Any: + pass + +@hookspec(firstresult=True) +def get_mail_class() -> Any: + pass + +@hookspec(firstresult=True) +def get_celery_app() -> Any: + pass + +@hookspec(firstresult=True) +def get_task(obj) -> Any: + pass + +@hookspec(firstresult=True) +def chord(arg) -> Any: + pass + +@hookspec(firstresult=True) +def fix_ml_files(playground_number, current_project) -> Any: + pass + +@hookspec(firstresult=True) +def write_ml_source(playground, playground_number, current_project, filename, finalize) -> Any: + pass + +@hookspec(firstresult=True) +def ensure_training_loaded(interview) -> Any: + pass + +@hookspec(firstresult=True) +def manage_chat_logs(mode: int, kwargs: dict) -> None: + pass + +@hookspec(firstresult=True) +def manage_global_objects(mode: int, kwargs: dict) -> None: + pass + +@hookspec(firstresult=True) +def manage_email_server_objects(mode: int, kwargs: dict) -> None: + pass + +@hookspec(firstresult=True) +def manage_tts_objects(mode: int, kwargs: dict) -> None: + pass + +@hookspec(firstresult=True) +def get_chat_log_internal(chat_mode, yaml_filename, session_id, user_id, temp_user_id, secret, self_user_id, self_temp_id) -> Any: + pass + +@hookspec(firstresult=True) +def get_ml_info(varname, default_package, default_file) -> Any: + pass diff --git a/docassemble_base/docassemble/base/interview_cache.py b/docassemble_base/docassemble/base/interview_cache.py index b38b5ecc0..0e50bf3d9 100644 --- a/docassemble_base/docassemble/base/interview_cache.py +++ b/docassemble_base/docassemble/base/interview_cache.py @@ -1,9 +1,9 @@ from docassemble.base.error import DAException -import docassemble.base.parse +from docassemble.base.interview_source import interview_source_from_string +from docassemble.base.parse import Interview cache = {} - def get_interview(path): if path is None: raise DAException("Tried to load interview source with no path") @@ -11,9 +11,9 @@ def get_interview(path): the_interview = cache[path]['interview'] the_interview.from_cache = True else: - interview_source = docassemble.base.parse.interview_source_from_string(path) + interview_source = interview_source_from_string(path) interview_source.update() - the_interview = interview_source.get_interview() + the_interview = Interview(source=interview_source) the_interview.from_cache = False cache[interview_source.path] = {'index': interview_source.get_index(), 'interview': the_interview, 'source': interview_source} return the_interview @@ -24,7 +24,7 @@ def clear_cache(path): del cache[path] -def cache_valid(questionPath): - if questionPath in cache and cache[questionPath]['index'] == cache[questionPath]['source'].get_index(): +def cache_valid(question_path): + if question_path in cache and cache[question_path]['index'] == cache[question_path]['source'].get_index(): return True return False diff --git a/docassemble_base/docassemble/base/interview_source.py b/docassemble_base/docassemble/base/interview_source.py new file mode 100644 index 000000000..d40ce2e35 --- /dev/null +++ b/docassemble_base/docassemble/base/interview_source.py @@ -0,0 +1,311 @@ +# pylint: disable=attribute-defined-outside-init,missing-class-docstring,too-many-instance-attributes +import re +import os +import copy +import datetime +import platform +from jinja2.exceptions import TemplateError +from jinja2 import FileSystemLoader, select_autoescape, TemplateNotFound +from jinja2.environment import Environment +from docassemble.base.error import DANotFoundError, DAError +from docassemble.base.logger import logmessage +from docassemble.base.functions import ( + package_question_filename, + standard_question_filename, + get_config, +) +from docassemble.base.hooks import ( + absolute_filename, + get_configuration, + get_server_redis, +) +from . import __version__ as da_version + +da_arch = platform.machine() + +class DAFileSystemLoader(FileSystemLoader): + + def get_source(self, environment, template): + if ':' not in template: + return super().get_source(environment, template) + template_path = None + for the_filename in question_path_options(template): + if the_filename is not None: + template_path = the_filename + break + if template_path is None or not os.path.isfile(template_path): + raise TemplateNotFound(template) + fspath = os.fspath(os.path.dirname(template_path)) + if fspath not in self.searchpath: + self.searchpath.append(fspath) + mtime = os.path.getmtime(template_path) + with open(template_path, 'r', encoding='utf-8') as fp: + source = fp.read() + return source, template_path, lambda: mtime == os.path.getmtime(template_path) + +class InterviewSource: + + def __init__(self, **kwargs): + if not hasattr(self, 'package'): + self.package = kwargs.get('package', None) + self.language = kwargs.get('language', '*') + self.dialect = kwargs.get('dialect', None) + self.testing = kwargs.get('testing', False) + self.translating = kwargs.get('translating', False) + + def __le__(self, other): + return str(self) <= (str(other) if isinstance(other, InterviewSource) else other) + + def __ge__(self, other): + return str(self) >= (str(other) if isinstance(other, InterviewSource) else other) + + def __gt__(self, other): + return str(self) > (str(other) if isinstance(other, InterviewSource) else other) + + def __lt__(self, other): + return str(self) < (str(other) if isinstance(other, InterviewSource) else other) + + def __eq__(self, other): + return self is other + + def __ne__(self, other): + return self is not other + + def __str__(self): + if hasattr(self, 'path'): + return str(self.path) + return 'interviewsource' + + def __hash__(self): + if hasattr(self, 'path'): + return hash((self.path,)) + return hash(('interviewsource',)) + + def set_path(self, path): + self.path = path + + def get_name(self): + if ':' in self.path: + return self.path + return self.get_package() + ':data/questions/' + self.path + + def get_index(self): + the_index = get_server_redis().get('da:interviewsource:' + self.path) + if the_index is None: + # logmessage("Updating index from get_index for " + self.path) + the_index = get_server_redis().incr('da:interviewsource:' + self.path) + return the_index + + def update_index(self): + # logmessage("Updating index for " + self.path) + get_server_redis().incr('da:interviewsource:' + self.path) + + def set_filepath(self, filepath): + self.filepath = filepath + + def set_directory(self, directory): + self.directory = directory + + def set_content(self, content): + self.content = content + + def set_language(self, language): + self.language = language + + def set_dialect(self, dialect): + self.dialect = dialect + + def set_testing(self, testing): + self.testing = testing + + def set_package(self, package): + self.package = package + + def update(self, **kwargs): # pylint: disable=unused-argument + return True + + def get_modtime(self): + return self._modtime # pylint: disable=no-member + + def get_language(self): + return self.language + + def get_dialect(self): + return self.dialect + + def get_package(self): + return self.package + + def get_testing(self): + return self.testing + + def append(self, path): # pylint: disable=unused-argument + return None + + +class InterviewSourceString(InterviewSource): + + def __init__(self, **kwargs): + self.set_path(kwargs.get('path', None)) + self.set_directory(kwargs.get('directory', None)) + self.set_content(kwargs.get('content', None)) + self._modtime = datetime.datetime.now(tz=datetime.timezone.utc) + super().__init__(**kwargs) + + +class InterviewSourceFile(InterviewSource): + + def __init__(self, **kwargs): + self.playground = None + if 'filepath' in kwargs: + if kwargs['filepath'].__class__.__name__.endswith('SavedFile'): + self.playground = kwargs['filepath'] + if self.playground.subdir and self.playground.subdir != 'default': + self.playground_file = os.path.join(self.playground.subdir, self.playground.filename) + else: + self.playground_file = self.playground.filename + # logmessage("The path is " + repr(self.playground.path)) + if os.path.isfile(self.playground.path) and os.access(self.playground.path, os.R_OK): + self.set_filepath(self.playground.path) + else: + logmessage("Details of playground path reference:") + logmessage("Keyword arguments were " + repr(kwargs)) + for attribute in ['file_number', 'fixed', 'section', 'filename', 'extension', 'directory', 'path', 'modtimes', 'keydict', 'subdir']: + if hasattr(self.playground, attribute): + logmessage(attribute + " is " + repr(getattr(self.playground, attribute))) + else: + logmessage(attribute + " did not exist") + if os.path.exists(self.playground.path): + if os.path.isfile(self.playground.path): + if os.access(self.playground.path, os.R_OK): + logmessage("path is a file and is readable") + else: + logmessage("path is a file but is not readable") + else: + logmessage("path was not a file") + else: + logmessage("path did not exist") + raise DANotFoundError("Reference to invalid playground path.") + else: + self.set_filepath(kwargs['filepath']) + else: + self.filepath = None + if 'path' in kwargs: + self.set_path(kwargs['path']) + super().__init__(**kwargs) + + def set_path(self, path): + self.path = path + parts = path.split(":") + if len(parts) == 2: + self.package = parts[0] + self.basename = parts[1] + else: + self.package = None + # if self.package is None: + # m = re.search(r'^/(playground\.[0-9]+)/', path) + # if m: + # self.package = m.group(1) + if self.filepath is None: + self.set_filepath(interview_source_from_string(self.path)) + if self.package is None and re.search(r'docassemble.base.data.', self.filepath): + self.package = 'docassemble.base' + + def set_filepath(self, filepath): + # logmessage("Called set_filepath with " + str(filepath)) + self.filepath = filepath + if self.filepath is None: + self.directory = None + else: + self.set_directory(os.path.dirname(self.filepath)) + + def reset_modtime(self): + try: + with open(self.filepath, 'a', encoding='utf-8'): + os.utime(self.filepath, None) + except: + logmessage("InterviewSourceFile: could not reset modification time on interview") + + def update(self, **kwargs): + try: + with open(self.filepath, 'r', encoding='utf-8') as the_file: + orig_text = the_file.read() + except: + return False + if not orig_text.startswith('# use jinja'): + self.set_content(orig_text) + return True + env = Environment( + loader=DAFileSystemLoader(self.directory), + autoescape=select_autoescape() + ) + if kwargs.get('raise_jinja_errors', True): + template = env.get_template(os.path.basename(self.filepath)) + else: + try: + template = env.get_template(os.path.basename(self.filepath)) + except TemplateError: + self.set_content(orig_text) + return True + data = copy.deepcopy(get_config('jinja data')) + data['__config__'] = copy.deepcopy(get_configuration()) + data['__version__'] = da_version + data['__architecture__'] = da_arch + data['__filename__'] = self.path + data['__current_package__'] = self.package + data['__parent_filename__'] = kwargs.get('parent_source', self).path + data['__parent_package__'] = kwargs.get('parent_source', self).package + data['__interview_filename__'] = kwargs.get('interview_source', self).path + data['__interview_package__'] = kwargs.get('interview_source', self).package + data['__hostname__'] = get_config('external hostname', None) or 'localhost' + data['__debug__'] = bool(get_config('debug', True)) + try: + self.set_content(template.render(data)) + except BaseException as err: + self.set_content("__error__: " + repr("Jinja2 rendering error: " + err.__class__.__name__ + ": " + str(err))) + return True + + def get_modtime(self): + # logmessage("get_modtime called in parse where path is " + str(self.path)) + if self.playground is not None: + return self.playground.get_modtime(filename=self.playground_file) + self._modtime = os.path.getmtime(self.filepath) + return self._modtime + + def append(self, path): + new_file = os.path.join(self.directory, path) + if os.path.isfile(new_file) and os.access(new_file, os.R_OK): + new_source = InterviewSourceFile() + new_source.path = path + new_source.directory = self.directory + new_source.basename = path + new_source.filepath = new_file + new_source.playground = self.playground + if hasattr(self, 'package'): + new_source.package = self.package + if new_source.update(): + return new_source + return None + +def question_path_options(path): + n = 0 + while n < 3: + if n == 0: + yield package_question_filename(path) + elif n == 1: + yield standard_question_filename(path) + elif n == 2: + yield absolute_filename(path) + n += 1 + +def interview_source_from_string(path, **kwargs): + if path is None: + raise DAError("Passed None to interview_source_from_string") + # logmessage("Trying to find " + path) + path = re.sub(r'(docassemble.playground[0-9]+[^:]*:)data/questions/(.*)', r'\1\2', path) + for the_filename in question_path_options(path): + if the_filename is not None: + new_source = InterviewSourceFile(filepath=the_filename, path=path) + if new_source.update(**kwargs): + return new_source + raise DANotFoundError("Interview " + str(path) + " not found") diff --git a/docassemble_base/docassemble/base/jinja.py b/docassemble_base/docassemble/base/jinja.py new file mode 100644 index 000000000..43afa012c --- /dev/null +++ b/docassemble_base/docassemble/base/jinja.py @@ -0,0 +1,570 @@ +import re +import os +from collections import abc, namedtuple +from itertools import groupby, chain +from docxtpl import RichText, DocxTemplate +from jinja2 import ChainableUndefined +from jinja2 import meta as jinja2meta +from jinja2.environment import Environment +from jinja2.ext import Extension +from jinja2.lexer import Token +from jinja2.runtime import StrictUndefined, UndefinedError +from jinja2.utils import internalcode, missing, object_type_repr +from .error import DAError, DASourceError, DAAttributeError, DAIndexError +from .filter.docx import inline_markdown_to_docx, markdown_to_docx +from .filter.utils import sanitize_xml +from .functions import ( + redact, + phone_number_in_e164, + manual_line_breaks, + bold, + single_to_double_newlines, + single_paragraph, + alpha, + qr_code, + country_name, + phone_number_formatted, + verbatim, + roman, + italic, +) +from .helpers import extract_missing_name, fix_quotes +from .language.capitalization import capitalize +from .language.currency import currency +from .language.language import ( + salutation, + add_separators, + comma_and_list, + title_case, + comma_list, +) +from .language.numbers import ordinal_number, nice_number, ordinal +from .language.utils import fix_punctuation +from .language.words import word +from .thread_context import this_thread +from .dates import ( + month_of, + format_date, + day_of, + format_time, + format_datetime, + year_of, + dow_of, +) + +NoneType = type(None) + +class DAExtension(Extension): + + def parse(self, parser): + raise NotImplementedError() + + def filter_stream(self, stream): + # in_var = False + met_pipe = False + for token in stream: + if token.type == 'variable_begin': + # in_var = True + met_pipe = False + if token.type == 'variable_end': + # in_var = False + if not met_pipe: + yield Token(token.lineno, 'pipe', None) + yield Token(token.lineno, 'name', 'ampersand_filter') + # if in_var and token.type == 'pipe': + # met_pipe = True + yield token + + +def ampersand_filter(value): + if value.__class__.__name__ in ('DAFile', 'DALink', 'DAStaticFile', 'DAFileCollection', 'DAFileList'): + return value + if value.__class__.__name__ in ('CustomInlineImage', 'InlineImage', 'RichText', 'Listing', 'Document', 'Subdoc', 'DALazyTemplate', 'Markup'): + return str(value) + if isinstance(value, (int, bool, float, NoneType)): + return value + if not isinstance(value, str): + value = str(value) + value = sanitize_xml(value) + if '' in value or '' in value: + return re.sub(r'&(?!#?[0-9A-Za-z]+;)', '&', value) + for auto_filter in this_thread.misc.get('auto jinja filter', []): + value = auto_filter(value) + return re.sub(r'>', '>', re.sub(r'<', '<', re.sub(r'&(?!#?[0-9A-Za-z]+;)', '&', value))) + + +class DAStrictUndefined(StrictUndefined): + __slots__ = ('_undefined_type',) + + def __init__(self, hint=None, obj=missing, name=None, exc=UndefinedError, accesstype=None): # pylint: disable=super-init-not-called + self._undefined_hint = hint + self._undefined_obj = obj + self._undefined_name = name + self._undefined_exception = exc + self._undefined_type = accesstype + + @internalcode + def __getattr__(self, name): + if name[:2] == '__': + raise AttributeError(name) + return self._fail_with_undefined_error(attribute=True) + + @internalcode + def __getitem__(self, index): + if index[:2] == '__': + raise IndexError(index) + return self._fail_with_undefined_error(item=True) + + @internalcode + def _fail_with_undefined_error(self, *args, **kwargs): + if self._undefined_obj is missing: + hint = "'%s' is undefined" % self._undefined_name + elif self._undefined_type == 'item' and hasattr(self._undefined_obj, 'instanceName'): + hint = "'%s[%r]' is undefined" % ( + self._undefined_obj.instanceName, + self._undefined_name + ) + elif 'attribute' in kwargs or self._undefined_type == 'attribute': + if hasattr(self._undefined_obj, 'instanceName'): + hint = "'%s.%s' is undefined" % ( + self._undefined_obj.instanceName, + self._undefined_name + ) + else: + hint = '%r has no attribute %r' % ( + object_type_repr(self._undefined_obj), + self._undefined_name + ) + else: + if hasattr(self._undefined_obj, 'instanceName'): + hint = "'%s[%r]' is undefined" % ( + self._undefined_obj.instanceName, + self._undefined_name + ) + else: + hint = '%s has no element %r' % ( + object_type_repr(self._undefined_obj), + self._undefined_name + ) + raise self._undefined_exception(hint) + __add__ = __radd__ = __mul__ = __rmul__ = __div__ = __rdiv__ = \ + __truediv__ = __rtruediv__ = __floordiv__ = __rfloordiv__ = \ + __mod__ = __rmod__ = __pos__ = __neg__ = __call__ = \ + __lt__ = __le__ = __gt__ = __ge__ = __int__ = \ + __float__ = __complex__ = __pow__ = __rpow__ = __sub__ = \ + __rsub__ = __iter__ = __str__ = __len__ = __nonzero__ = __eq__ = \ + __ne__ = __bool__ = __hash__ = _fail_with_undefined_error + + +class DASkipUndefined(ChainableUndefined): + """Undefined handler for Jinja2 exceptions that allows rendering most + templates that have undefined variables. It will not fix all broken + templates. For example, if the missing variable is used in a complex + mathematical expression it may still break (but expressions with only two + elements should render as ''). + """ + + def __init__(self, *pargs, **kwargs): # pylint: disable=super-init-not-called + # Handle the way Docassemble DAEnvironment triggers attribute errors + pass + + def __str__(self) -> str: + return '' + + def __call__(self, *pargs, **kwargs) -> "DASkipUndefined": + return self + + __getitem__ = __getattr__ = __call__ + + def __eq__(self, *pargs) -> bool: + return False + + # need to return a bool type + __bool__ = __ne__ = __le__ = __lt__ = __gt__ = __ge__ = __nonzero__ = __eq__ + + # let undefined variables work in for loops + + def __iter__(self, *pargs) -> "DASkipUndefined": + return self + + def __next__(self, *pargs) -> None: + raise StopIteration + + # need to return an int type + + def __int__(self, *pargs) -> int: + return 0 + + __len__ = __int__ + + # need to return a float type + + def __float__(self, *pargs) -> float: + return 0.0 + + # need to return complex type + + def __complex__(self, *pargs) -> complex: + return 0j + + def __add__(self, *pargs, **kwargs) -> str: + return self.__str__() + + # type can be anything. we want it to work with `str()` function though + # and we do not want to silently give wrong math results. + # note that this means 1 + (undefined) or (undefined) + 1 will work but not 1 + (undefined) + 1 + __radd__ = __mul__ = __rmul__ = __div__ = __rdiv__ = \ + __truediv__ = __rtruediv__ = __floordiv__ = __rfloordiv__ = \ + __mod__ = __rmod__ = __pos__ = __neg__ = __pow__ = __rpow__ = \ + __sub__ = __rsub__ = __hash__ = __add__ + + +class DAEnvironment(Environment): + + def from_string(self, source, **kwargs): # pylint: disable=arguments-differ + source = re.sub(r'({[\%\{].*?[\%\}]})', fix_quotes, source) + return super().from_string(source, **kwargs) + + def getitem(self, obj, argument): + try: + return obj[argument] + except (DAAttributeError, DAIndexError) as err: + varname = extract_missing_name(err) + if 'pending_error' in this_thread.misc: + del this_thread.misc['pending_error'] + return self.undefined(obj=missing, name=varname) + except (AttributeError, TypeError, LookupError): + if 'pending_error' in this_thread.misc: + del this_thread.misc['pending_error'] + return self.undefined(obj=obj, name=argument, accesstype='item') + + def getattr(self, obj, attribute): + try: + return getattr(obj, attribute) + except DAAttributeError as err: + if 'pending_error' in this_thread.misc: + del this_thread.misc['pending_error'] + varname = extract_missing_name(err) + return self.undefined(obj=missing, name=varname) + except AttributeError: + if 'pending_error' in this_thread.misc: + del this_thread.misc['pending_error'] + return self.undefined(obj=obj, name=attribute, accesstype='attribute') + + +def mygetattr(y, attr, default=None): + for attribute in attr.split('.'): + y = getattr(y, attribute, default) + return y + + +def str_or_original(y, case_sensitive): + if case_sensitive: + if hasattr(y, 'instanceName'): + if y.__class__.__name__ in ('Value', 'PeriodicValue'): + return y.amount() + return str(y) + return y + if hasattr(y, 'instanceName'): + if y.__class__.__name__ in ('Value', 'PeriodicValue'): + return y.amount() + return str(y).lower() + try: + return y.lower() + except: + return y + + +def dictsort_filter(dictionary, case_sensitive=False, by='key', reverse=False): + if by == 'value': + return sorted(dictionary.items(), key=lambda y: str_or_original(y[1], case_sensitive), reverse=reverse) + return sorted(dictionary.items(), key=lambda y: str_or_original(y[0], case_sensitive), reverse=reverse) + + +def sort_filter(the_array, reverse=False, case_sensitive=False, attribute=None): + if attribute is None: + if not case_sensitive: + def key_func(y): + return str_or_original(y, case_sensitive) + else: + key_func = None + else: + if isinstance(attribute, list): + attributes = [str(y).strip() for y in attribute] + else: + attributes = [y.strip() for y in str(attribute).split(',')] + def key_func(y): + return [str_or_original(mygetattr(y, attribute), case_sensitive) for attribute in attributes] + return sorted(the_array, key=key_func, reverse=reverse) + +_GroupTuple = namedtuple('_GroupTuple', ['grouper', 'list']) +_GroupTuple.__repr__ = tuple.__repr__ +_GroupTuple.__str__ = tuple.__str__ + + +def groupby_filter(the_array, attr_name): + + def func(y): + return mygetattr(y, attr_name) + return [_GroupTuple(key, list(values)) for key, values in groupby(sorted(the_array, key=func), func)] + + +def max_filter(the_array, case_sensitive=False, attribute=None): + it = iter(the_array) + try: + first = next(it) + except StopIteration: + raise DAError("max: list was empty") + if attribute: + def key_func(y): + return str_or_original(mygetattr(y, attribute), case_sensitive=case_sensitive) + else: + def key_func(y): + return str_or_original(y, case_sensitive=case_sensitive) + return max(chain([first], it), key=key_func) + + +def min_filter(the_array, case_sensitive=False, attribute=None): + it = iter(the_array) + try: + first = next(it) + except StopIteration: + raise DAError("min: list was empty") + if attribute: + def key_func(y): + return str_or_original(mygetattr(y, attribute), case_sensitive=case_sensitive) + else: + def key_func(y): + return str_or_original(y, case_sensitive=case_sensitive) + return min(chain([first], it), key=key_func) + + +def sum_filter(the_array, attribute=None, start=0): + if attribute is not None: + the_array = [mygetattr(y, attribute) for y in the_array] + return sum(the_array, start) + + +def unique_filter(the_array, case_sensitive=False, attribute=None): + seen = set() + if attribute is None: + for item in the_array: + new_item = str_or_original(item, case_sensitive) + if new_item not in seen: + seen.add(new_item) + yield item + else: + for item in the_array: + new_item = str_or_original(mygetattr(item, attribute), case_sensitive) + if new_item not in seen: + seen.add(new_item) + yield mygetattr(item, attribute) + + +def join_filter(the_array, d="", attribute=None): + if attribute is not None: + return d.join([str(mygetattr(y, attribute)) for y in the_array]) + return d.join([str(y) for y in the_array]) + + +def attr_filter(var, attr_name): + return mygetattr(var, attr_name) + + +def selectattr_filter(*pargs, **kwargs): + if len(pargs) > 2: + the_array = pargs[0] + attr_name = pargs[1] + func_name = pargs[2] + env = custom_jinja_env() + def func(item): + return env.call_test(func_name, item, pargs[3:], kwargs) + for item in the_array: + if func(mygetattr(item, attr_name)): + yield item + else: + for item in pargs[0]: + if mygetattr(item, pargs[1]): + yield item + + +def rejectattr_filter(*pargs, **kwargs): + if len(pargs) > 2: + the_array = pargs[0] + attr_name = pargs[1] + func_name = pargs[2] + env = custom_jinja_env() + def func(item): + return env.call_test(func_name, item, pargs[3:], kwargs) + for item in the_array: + if not func(mygetattr(item, attr_name)): + yield item + else: + for item in pargs[0]: + if not mygetattr(item, pargs[1]): + yield item + + +def chain_filter(*pargs, **kwargs): # pylint: disable=unused-argument + the_list = [] + for parg in pargs: + if isinstance(parg, str): + the_list.append(parg) + elif (hasattr(parg, 'instanceName') and hasattr(parg, 'elements')): + if isinstance(parg.elements, dict): + for sub_parg in parg.values(): + the_list.append(sub_parg) + else: + for sub_parg in parg: + the_list.append(sub_parg) + elif isinstance(parg, abc.Iterable): + for sub_parg in parg: + the_list.append(sub_parg) + else: + the_list.append(parg) + return chain(*the_list) + + +def map_filter(*pargs, **kwargs): + if len(pargs) >= 2: + the_array = pargs[0] + the_filter = pargs[1] + env = custom_jinja_env() + if the_filter not in env.filters: + raise DAError('filter passed to map() does not exist') + for item in the_array: + yield env.call_filter(the_filter, item, pargs[2:], kwargs) + else: + if 'attribute' in kwargs: + if 'default' in kwargs: + for item in pargs[0]: + yield mygetattr(item, kwargs['attribute'], kwargs['default']) + else: + for item in pargs[0]: + yield mygetattr(item, kwargs['attribute']) + elif 'index' in kwargs: + if 'default' in kwargs: + for item in pargs[0]: + yield item.get(kwargs['index'], kwargs['default']) + else: + for item in pargs[0]: + yield item[kwargs['index']] + elif 'function' in kwargs: + the_kwargs = kwargs.get('kwargs', {}) + the_pargs = kwargs.get('pargs', []) + if not isinstance(the_kwargs, dict): + raise DAError('kwargs passed to map() must be a dictionary') + if not isinstance(the_pargs, list): + raise DAError('pargs passed to map() must be a list') + for item in pargs[0]: + yield kwargs['function'](item, *the_pargs, **the_kwargs) + else: + raise DAError('map() must refer to a function, index, attribute, or filter') + + +def markdown_filter(text): + return markdown_to_docx(text, this_thread.current_question, this_thread.misc.get('docx_template', None)) + + +def inline_markdown_filter(text): + return inline_markdown_to_docx(text, this_thread.current_question, this_thread.misc.get('docx_template', None)) + + +def get_builtin_jinja_filters(): + return { + 'ampersand_filter': ampersand_filter, + 'markdown': markdown_filter, + 'add_separators': add_separators, + 'inline_markdown': inline_markdown_filter, + 'paragraphs': single_to_double_newlines, + 'manual_line_breaks': manual_line_breaks, + 'RichText': RichText, + 'groupby': groupby_filter, + 'max': max_filter, + 'min': min_filter, + 'sum': sum_filter, + 'unique': unique_filter, + 'join': join_filter, + 'attr': attr_filter, + 'selectattr': selectattr_filter, + 'rejectattr': rejectattr_filter, + 'sort': sort_filter, + 'dictsort': dictsort_filter, + 'format_date': format_date, + 'format_datetime': format_datetime, + 'format_time': format_time, + 'month_of': month_of, + 'year_of': year_of, + 'day_of': day_of, + 'dow_of': dow_of, + 'qr_code': qr_code, + 'nice_number': nice_number, + 'ordinal': ordinal, + 'ordinal_number': ordinal_number, + 'currency': currency, + 'comma_list': comma_list, + 'comma_and_list': comma_and_list, + 'capitalize': capitalize, + 'salutation': salutation, + 'alpha': alpha, + 'roman': roman, + 'word': word, + 'bold': bold, + 'italic': italic, + 'title_case': title_case, + 'single_paragraph': single_paragraph, + 'phone_number_formatted': phone_number_formatted, + 'phone_number_in_e164': phone_number_in_e164, + 'country_name': country_name, + 'fix_punctuation': fix_punctuation, + 'redact': redact, + 'verbatim': verbatim, + 'map': map_filter, + 'chain': chain_filter, + 'any': any, + 'all': all + } + + +registered_jinja_filters = {} + + +def custom_jinja_env(skip_undefined=False): + if skip_undefined: + env = DAEnvironment(undefined=DASkipUndefined, extensions=[DAExtension]) + else: + env = DAEnvironment(undefined=DAStrictUndefined, extensions=[DAExtension]) + env.filters.update(registered_jinja_filters) + env.filters.update(get_builtin_jinja_filters()) + return env + + +def register_jinja_filter(filter_name, func): + if filter_name in get_builtin_jinja_filters(): + raise DAError("Cannot register filter with same name as built-in filter %s" % filter_name) + registered_jinja_filters[filter_name] = func + + +def get_docx_variables(the_path): + names = set() + if not os.path.isfile(the_path): + raise DASourceError("Missing docx template file " + os.path.basename(the_path)) + try: + docx_template = DocxTemplate(the_path) + docx_template.render_init() + the_env = custom_jinja_env() + the_xml = docx_template.get_xml() + the_xml = re.sub(r'])', r'\n 1: + return a[0].upper() + a[1:] + return a + +capitalize = language_function_constructor('capitalize') + +update_language_function('*', 'capitalize', capitalize_default) diff --git a/docassemble_base/docassemble/base/language/control.py b/docassemble_base/docassemble/base/language/control.py new file mode 100644 index 000000000..5d73f5673 --- /dev/null +++ b/docassemble_base/docassemble/base/language/control.py @@ -0,0 +1,146 @@ +import locale +from docassemble.base.thread_context import this_thread +from docassemble.base.logger import logmessage + +def get_language(): + """Return the current language code. + + Returns: + str: The current language code (e.g., ``'en'``, ``'es'``). + """ + return this_thread.language + + +def set_language(lang, dialect=None, voice=None): + """Set the language used for linguistic functions and the web application. + + Does not change the Python locale; call ``update_locale()`` for that. + Should be called in an ``initial`` code block so it takes effect on every + page load. + + Args: + lang (str): A lowercase ISO-639-1 or ISO-639-3 language code + (e.g., ``'en'``, ``'es'``, ``'fr'``). + dialect (str, optional): A dialect code for the text-to-speech engine. + Defaults to None. + voice (str, optional): A voice name for the text-to-speech engine. + Defaults to None. + """ + try: + if dialect: + this_thread.dialect = dialect + elif lang != this_thread.language: + this_thread.dialect = None + except: + pass + try: + if voice: + this_thread.voice = voice + elif lang != this_thread.language: + this_thread.voice = None + except: + pass + this_thread.language = lang + + +def set_country(country): + """Set the current country used for phone number formatting and other locale features. + + Args: + country (str): A two-letter uppercase ISO 3166-1 alpha-2 country code + (e.g., ``'US'``, ``'GB'``, ``'DE'``). + """ + this_thread.country = country + + +def get_country(): + """Return the current country code. + + Returns: + str: A two-letter uppercase ISO 3166-1 alpha-2 country code + (e.g., ``'US'``). Defaults to ``'US'`` unless configured otherwise. + """ + return this_thread.country + + +def get_dialect(): + """Return the current dialect. + + Returns: + str: The dialect code set by the ``dialect`` keyword argument to + :func:`set_language`, or ``None`` if no dialect has been set. + """ + return this_thread.dialect + + +def get_voice(): + """Return the current voice. + + Returns: + str: The voice name set by the ``voice`` keyword argument to + :func:`set_language`, or ``None`` if no voice has been set. + """ + return this_thread.voice + + +def set_locale(*pargs, **kwargs): + """Set the current locale string and/or locale convention overrides. + + Calling ``set_locale('FR.utf8')`` stores the locale string so that + :func:`get_locale` returns it. The actual Python locale does not change + until :func:`update_locale` is called. Keyword arguments such as + ``currency_symbol`` override individual locale conventions used by + functions like :func:`currency` and :func:`currency_symbol`. + + Args: + *pargs: An optional locale string (e.g. ``'FR.utf8'``). + **kwargs: Locale convention overrides (e.g. ``currency_symbol='€'``). + """ + if len(pargs) == 1: + this_thread.locale = pargs[0] + if len(kwargs): + this_thread.misc['locale_overrides'] = kwargs + + +def get_locale(*pargs): + """Return the current locale setting or a specific locale convention. + + With no arguments, returns the locale string previously set with + :func:`set_locale`. With one argument, returns the value of the named + locale convention (e.g. ``'currency_symbol'``), taking into account any + overrides set with :func:`set_locale`. + + Args: + *pargs: An optional locale convention name (e.g. + ``'currency_symbol'``). + + Returns: + str or None: The locale string when called with no arguments, or the + value of the requested locale convention (``None`` if not found). + """ + if len(pargs) == 1: + if 'locale_overrides' in this_thread.misc and pargs[0] in this_thread.misc['locale_overrides']: + return this_thread.misc['locale_overrides'][pargs[0]] + return locale.localeconv().get(pargs[0], None) + return this_thread.locale + + +def update_locale(): + """Update the Python locale based on the current language and locale settings. + + Applies the locale string previously set with :func:`set_locale` (combined + with the current language from :func:`get_language` when necessary) so that + Python's ``locale`` module reflects the desired locale. This is required + for functions like :func:`currency` and :func:`currency_symbol` to produce + locale-appropriate formatting. + """ + if '_' in this_thread.locale: + the_locale = str(this_thread.locale) + else: + the_locale = str(this_thread.language) + '_' + str(this_thread.locale) + try: + locale.setlocale(locale.LC_ALL, the_locale) + except BaseException as err: + logmessage("update_locale error: unable to set the locale to " + the_locale) + logmessage(err.__class__.__name__ + ": " + str(err)) + locale.setlocale(locale.LC_ALL, 'en_US.utf8') diff --git a/docassemble_base/docassemble/base/language/core.py b/docassemble_base/docassemble/base/language/core.py new file mode 100644 index 000000000..fb767caf2 --- /dev/null +++ b/docassemble_base/docassemble/base/language/core.py @@ -0,0 +1,38 @@ +from typing import Callable +from jinja2.runtime import Undefined +from docassemble.base.logger import logmessage +from docassemble.base.thread_context import this_thread + +language_functions: dict[str, dict[str, Callable]] = {} + +def update_language_function(lang, term, func): + if term not in language_functions: + language_functions[term] = {} + language_functions[term][lang] = func + + +def ensure_definition(*pargs, **kwargs): + for val in pargs: + if isinstance(val, Undefined): + str(val) + for val in kwargs.values(): + if isinstance(val, Undefined): + str(val) + + +def language_function_constructor(term): + + def func(*args, **kwargs): + ensure_definition(*args, **kwargs) + language = kwargs.get('language', None) + if language is None: + language = this_thread.language + if language in language_functions[term]: + return language_functions[term][language](*args, **kwargs) + if '*' in language_functions[term]: + return language_functions[term]['*'](*args, **kwargs) + if 'en' in language_functions[term]: + logmessage("Term " + str(term) + " is not defined for language " + str(language)) + return language_functions[term]['en'](*args, **kwargs) + raise SystemError("term " + str(term) + " not defined in language_functions for English or *") + return func diff --git a/docassemble_base/docassemble/base/language/currency.py b/docassemble_base/docassemble/base/language/currency.py new file mode 100644 index 000000000..44a08671d --- /dev/null +++ b/docassemble_base/docassemble/base/language/currency.py @@ -0,0 +1,125 @@ +import locale +from docassemble.base.hooks import get_configuration +from docassemble.base.thread_context import this_thread +from .control import get_locale +from .core import ( + language_function_constructor, + ensure_definition, + update_language_function, + language_functions, +) + +currency = language_function_constructor('currency') +currency_symbol = language_function_constructor('currency_symbol') + +if currency.__doc__ is None: + currency.__doc__ = """Format a number as a currency value using the current locale. + + Args: + value: The numeric value to format. + **kwargs: Optional keyword arguments including ``decimals`` (bool, + default ``True``), ``symbol`` (str override for the currency + symbol), and ``symbol_precedes`` (bool controlling symbol + position). + + Returns: + str: The formatted currency string (e.g. ``currency(45.2)`` returns + ``'$45.20'`` for a US locale). + """ +if currency_symbol.__doc__ is None: + currency_symbol.__doc__ = """Return the currency symbol for the current locale. + + Returns: + str: The currency symbol (e.g. ``'$'`` for a US locale). Respects + overrides set via :func:`set_locale` or the ``currency symbol`` + configuration setting. + """ + +def currency_symbol_default(**kwargs): # pylint: disable=unused-argument + """Returns the currency symbol for the current locale.""" + return str(locale.localeconv()['currency_symbol']) + + +def currency_default(the_value, **kwargs): + """Returns the value as a currency, according to the conventions of + the current locale. Use the optional keyword argument + decimals=False if you do not want to see decimal places in the + number, and the optional currency_symbol for a different symbol + than the default. + + """ + decimals = kwargs.get('decimals', True) + symbol = kwargs.get('symbol', None) + symbol_precedes = kwargs.get('symbol_precedes', None) + ensure_definition(the_value, decimals, symbol) + obj_type = type(the_value).__name__ + if obj_type in ['FinancialList', 'PeriodicFinancialList']: + the_value = the_value.total() + elif obj_type in ['Value', 'PeriodicValue']: + if the_value.exists: + the_value = the_value.amount() + else: + the_value = 0 + elif obj_type == 'DACatchAll': + the_value = float(the_value) + try: + float(the_value) + except: + return '' + the_float_value = float(the_value) + the_symbol = None + if symbol is not None: + the_symbol = symbol + elif 'locale_overrides' in this_thread.misc and 'currency_symbol' in this_thread.misc['locale_overrides']: + the_symbol = this_thread.misc['locale_overrides']['currency_symbol'] + elif language_functions['currency_symbol']['*'] is not currency_symbol_default: + the_symbol = currency_symbol() + the_symbol_precedes = None + if symbol_precedes is not None: + the_symbol_precedes = symbol_precedes + elif 'locale_overrides' in this_thread.misc and the_float_value < 0 and 'n_cs_precedes' in this_thread.misc['locale_overrides']: + the_symbol_precedes = bool(this_thread.misc['locale_overrides']['n_cs_precedes']) + elif 'locale_overrides' in this_thread.misc and 'p_cs_precedes' in this_thread.misc['locale_overrides']: + the_symbol_precedes = bool(this_thread.misc['locale_overrides']['p_cs_precedes']) + if the_symbol is None and the_symbol_precedes is None and decimals: + return str(locale.currency(the_float_value, symbol=True, grouping=True)) + if the_symbol is None: + the_symbol = currency_symbol() + if the_symbol_precedes is None: + if the_float_value < 0: + the_symbol_precedes = bool(get_locale('n_cs_precedes')) + else: + the_symbol_precedes = bool(get_locale('p_cs_precedes')) + output = '' + if the_symbol_precedes: + output += the_symbol + if the_float_value < 0: + if get_locale('n_sep_by_space'): + output += ' ' + elif get_locale('p_sep_by_space'): + output += ' ' + if decimals: + output += locale.format_string('%.' + str(get_configuration().get('currency decimal places', locale.localeconv()['frac_digits'])) + 'f', the_float_value, grouping=True, monetary=True) + else: + output += locale.format_string("%d", int(the_float_value), grouping=True, monetary=True) + if not the_symbol_precedes: + if the_float_value < 0: + if get_locale('n_sep_by_space'): + output += ' ' + elif get_locale('p_sep_by_space'): + output += ' ' + output += the_symbol + return output + + +def get_currency_symbol(): + """Returns the current setting for the currency symbol if there is + one, and otherwise returns the default currency symbol. + + """ + if 'locale_overrides' in this_thread.misc and 'currency_symbol' in this_thread.misc['locale_overrides']: + return this_thread.misc['locale_overrides']['currency_symbol'] + return currency_symbol() + + +update_language_function('*', 'currency_symbol', currency_symbol_default) diff --git a/docassemble_base/docassemble/base/language/language.py b/docassemble_base/docassemble/base/language/language.py new file mode 100644 index 000000000..466d3ba55 --- /dev/null +++ b/docassemble_base/docassemble/base/language/language.py @@ -0,0 +1,747 @@ +import titlecase +from .core import ensure_definition, language_function_constructor, language_functions +from .capitalization import capitalize +from .words import word +from .numbers import nice_number, nice_number_default +from .language_en import ( + verb_present_en, + verb_past_en, + possessify_en, + indefinite_article_en, + noun_singular_en, + noun_plural_en, + add_separators_en, + comma_list_en, + comma_and_list_en, +) +from .language_es import ( + verb_past_es, + noun_plural_es, + indefinite_article_es, + noun_singular_es, + verb_present_es, + comma_and_list_es, +) +from .language_de import ( + indefinite_article_de, + noun_singular_de, + verb_past_de, + verb_present_de, + noun_plural_de, + comma_and_list_de, +) +from .language_fr import verb_past_fr, noun_plural_fr, noun_singular_fr, verb_present_fr +from .language_it import ( + noun_singular_it, + noun_plural_it, + verb_past_it, + indefinite_article_it, + verb_present_it, +) +from .language_nl import verb_past_nl, verb_present_nl, noun_singular_nl, noun_plural_nl +from .currency import currency_default +from .capitalization import capitalize_default + +def titlecasestr(text): + return titlecase.titlecase(str(text)) + + +def salutation_default(indiv, **kwargs): + """Returns Mr., Ms., etc. for an individual.""" + with_name = kwargs.get('with_name', False) + with_name_and_punctuation = kwargs.get('with_name_and_punctuation', False) + ensure_definition(indiv, with_name, with_name_and_punctuation) + used_gender = False + if hasattr(indiv, 'salutation_to_use') and indiv.salutation_to_use is not None: + salut = indiv.salutation_to_use + elif hasattr(indiv, 'is_doctor') and indiv.is_doctor: + salut = 'Dr.' + elif hasattr(indiv, 'is_judge') and indiv.is_judge: + salut = 'Judge' + elif hasattr(indiv, 'name') and hasattr(indiv.name, 'suffix') and indiv.name.suffix in ('MD', 'PhD'): + salut = 'Dr.' + elif hasattr(indiv, 'name') and hasattr(indiv.name, 'suffix') and indiv.name.suffix == 'J': + salut = 'Judge' + elif indiv.gender == 'female': + used_gender = True + salut = 'Ms.' + else: + used_gender = True + salut = 'Mr.' + if with_name_and_punctuation or with_name: + if used_gender and indiv.gender not in ('male', 'female'): + salut_and_name = indiv.name.full() + else: + salut_and_name = salut + ' ' + indiv.name.last + if with_name_and_punctuation: + if hasattr(indiv, 'is_friendly') and indiv.is_friendly: + punct = ',' + else: + punct = ':' + return salut_and_name + punct + if with_name: + return salut_and_name + return salut + + +def quantity_noun_default(the_number, noun, **kwargs): + as_integer = kwargs.get('as_integer', True) + capitalize_arg = kwargs.get('capitalize', False) + language = kwargs.get('language', None) + ensure_definition(the_number, noun, as_integer, capitalize_arg, language) + if as_integer: + the_number = int(round(the_number)) + result = nice_number(the_number, language=language) + " " + noun_plural(noun, the_number, language=language) + if capitalize_arg: + return capitalize(result) + return result + + +def prefix_constructor(prefix): + + def func(the_word, **kwargs): + ensure_definition(the_word, **kwargs) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(str(prefix)) + str(the_word) + return str(prefix) + str(the_word) + return func + + +def double_prefix_constructor_reverse(prefix_one, prefix_two): + + def func(word_one, word_two, **kwargs): + ensure_definition(word_one, word_two, **kwargs) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(str(prefix_one)) + str(word_two) + str(prefix_two) + str(word_one) + return str(prefix_one) + str(word_two) + str(prefix_two) + str(word_one) + return func + + +def prefix_constructor_two_arguments(prefix, **kwargs): # pylint: disable=unused-argument + + def func(word_one, word_two, **kwargs): + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(str(prefix)) + str(word_one) + ' ' + str(word_two) + return str(prefix) + str(word_one) + ' ' + str(word_two) + return func + + +def middle_constructor(middle, **kwargs): # pylint: disable=unused-argument + + def func(a, b, **kwargs): + ensure_definition(a, b, **kwargs) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(str(a)) + str(middle) + str(b) + return str(a) + str(middle) + str(b) + return func + + +def a_preposition_b_default(a, b, **kwargs): + ensure_definition(a, b, **kwargs) + if hasattr(a, 'preposition'): + preposition = word(a.preposition) + else: + preposition = word('in the') + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(str(a)) + str(' ' + preposition + ' ') + str(b) + return str(a) + str(' ' + preposition + ' ') + str(b) + + + +in_the = language_function_constructor('in_the') +a_preposition_b = language_function_constructor('a_preposition_b') +a_in_the_b = language_function_constructor('a_in_the_b') +i_subjective = language_function_constructor('i_subjective') +he_subjective = language_function_constructor('he_subjective') +she_subjective = language_function_constructor('she_subjective') +genderless_subjective = language_function_constructor('genderless_subjective') +myself = language_function_constructor('myself') +itself = language_function_constructor('itself') +herself = language_function_constructor('herself') +himself = language_function_constructor('himself') +themselves = language_function_constructor('themselves') +genderless_self = language_function_constructor('genderless_self') +yourself = language_function_constructor('yourself') +yourselves = language_function_constructor('yourselves') +ourselves = language_function_constructor('ourselves') +you_subjective = language_function_constructor('you_subjective') +you_subjective_plural = language_function_constructor('you_subjective_plural') +we_subjective = language_function_constructor('we_subjective') +they_subjective = language_function_constructor('they_subjective') +it_subjective = language_function_constructor('it_subjective') +it_objective = language_function_constructor('it_objective') +them_objective = language_function_constructor('them_objective') +genderless_objective = language_function_constructor('genderless_objective') +me_objective = language_function_constructor('me_objective') +him_objective = language_function_constructor('him_objective') +her_objective = language_function_constructor('her_objective') +our_objective = language_function_constructor('our_objective') +you_objective = language_function_constructor('you_objective') +you_objective_plural = language_function_constructor('you_objective_plural') +us_objective = language_function_constructor('us_objective') +are_we = language_function_constructor('are_we') +are_you = language_function_constructor('are_you') +are_you_plural = language_function_constructor('are_you_plural') +am_i = language_function_constructor('am_i') +her = language_function_constructor('her') +his = language_function_constructor('his') +are_word = language_function_constructor('are_word') +is_word = language_function_constructor('is_word') +their = language_function_constructor('their') +my_possessive = language_function_constructor('my_possessive') +our_possessive = language_function_constructor('our_possessive') +of_the = language_function_constructor('of_the') +your = language_function_constructor('your') +your_plural = language_function_constructor('your_plural') +some = language_function_constructor('some') +its = language_function_constructor('its') +the = language_function_constructor('the') +these = language_function_constructor('these') +this = language_function_constructor('this') +does_a_b = language_function_constructor('does_a_b') +do_a_b = language_function_constructor('do_a_b') +did_a_b = language_function_constructor('did_a_b') +did_a_b_plural = language_function_constructor('did_a_b_plural') +do_i = language_function_constructor('do_i') +do_we = language_function_constructor('do_we') +do_you = language_function_constructor('do_you') +do_you_plural = language_function_constructor('do_you_plural') +did_i = language_function_constructor('did_i') +did_we = language_function_constructor('did_we') +did_you = language_function_constructor('did_you') +did_you_plural = language_function_constructor('did_you_plural') +was_i = language_function_constructor('was_i') +were_we = language_function_constructor('were_we') +were_you = language_function_constructor('were_you') +were_you_plural = language_function_constructor('were_you_plural') +was_a_b = language_function_constructor('was_a_b') +were_a_b = language_function_constructor('were_a_b') +were_a_b_plural = language_function_constructor('were_a_b_plural') +have_i = language_function_constructor('have_i') +have_we = language_function_constructor('have_we') +have_you = language_function_constructor('have_you') +have_you_plural = language_function_constructor('have_you_plural') +has_a_b = language_function_constructor('has_a_b') +have_a_b = language_function_constructor('have_a_b') +verb_past = language_function_constructor('verb_past') +verb_present = language_function_constructor('verb_present') +noun_plural = language_function_constructor('noun_plural') +noun_singular = language_function_constructor('noun_singular') +indefinite_article = language_function_constructor('indefinite_article') +period_list = language_function_constructor('period_list') +name_suffix = language_function_constructor('name_suffix') +possessify = language_function_constructor('possessify') +possessify_long = language_function_constructor('possessify_long') +comma_list = language_function_constructor('comma_list') +comma_and_list = language_function_constructor('comma_and_list') +add_separators = language_function_constructor('add_separators') +quantity_noun = language_function_constructor('quantity_noun') +title_case = language_function_constructor('title_case') +salutation = language_function_constructor('salutation') + +if verb_past.__doc__ is None: + verb_past.__doc__ = """Return the past tense of a verb. + + Args: + verb (str): The verb to conjugate. + **kwargs: Optional conjugation parameters passed to the underlying + language function (e.g. ``'3gp'`` for third-person past tense). + + Returns: + str: The past-tense form of the verb (e.g. ``verb_past('help')`` + returns ``'helped'``). + """ +if verb_present.__doc__ is None: + verb_present.__doc__ = """Return the present tense of a verb. + + Args: + verb (str): The verb to conjugate (may be in any tense). + **kwargs: Optional conjugation parameters passed to the underlying + language function (e.g. ``'3sg'`` for third-person singular). + + Returns: + str: The present-tense form of the verb (e.g. + ``verb_present('helped', '3sg')`` returns ``'helps'``). + """ +if noun_plural.__doc__ is None: + noun_plural.__doc__ = """Return the plural form of a noun. + + Args: + noun (str): The noun to pluralize. + *pargs: An optional quantity (number, list, dict, or set). When the + quantity is exactly ``1`` the singular form is returned instead. + **kwargs: Pass ``noun_is_singular=True`` to skip singularization + before pluralizing. + + Returns: + str: The plural form of the noun, or the singular form if the + optional quantity equals ``1``. + """ +if noun_singular.__doc__ is None: + noun_singular.__doc__ = """Return the singular form of a noun. + + Args: + noun (str): The noun to singularize. + *pargs: An optional quantity (number, list, dict, or set). When the + quantity is not ``1`` the original noun is returned unchanged. + + Returns: + str: The singular form of the noun, or the original noun when the + optional quantity is not ``1``. + """ +if indefinite_article.__doc__ is None: + indefinite_article.__doc__ = """Return a noun preceded by the appropriate indefinite article. + + Args: + noun (str): The noun phrase to precede with an article. + **kwargs: Additional keyword arguments passed to the underlying + language function. + + Returns: + str: The noun prefixed with ``'a'`` or ``'an'`` as appropriate + (e.g. ``indefinite_article('apple')`` returns ``'an apple'``). + """ +if capitalize.__doc__ is None: + capitalize.__doc__ = """Return the input string with the first letter capitalized. + + Args: + a (str): The string to capitalize. + **kwargs: Additional keyword arguments passed to the underlying + language function. + + Returns: + str: The input string with its first character converted to + upper case. + """ +if period_list.__doc__ is None: + period_list.__doc__ = """Return a list of per-year period options for use in multiple-choice fields. + + Returns: + list: A list of ``[number, label]`` pairs representing common + payment periods (e.g. ``[[12, 'Per Month'], [1, 'Per Year'], + [52, 'Per Week'], ...]``). + """ +if name_suffix.__doc__ is None: + name_suffix.__doc__ = """Return a list of common name suffixes for use in multiple-choice fields. + + Returns: + list: A list of name suffix strings such as + ``['Jr', 'Sr', 'II', 'III', 'IV', 'V', 'VI']``. + """ +if possessify.__doc__ is None: + possessify.__doc__ = """Return the possessive phrase combining two arguments. + + Args: + a: The possessor. + b: The thing possessed. + **kwargs: Additional keyword arguments passed to the underlying + language function. + + Returns: + str: A possessive phrase such as ``"a's b"``. + """ +if possessify_long.__doc__ is None: + possessify_long.__doc__ = """Return the long possessive phrase combining two arguments. + + Args: + a: The possessor. + b: The thing possessed. + **kwargs: Additional keyword arguments passed to the underlying + language function. + + Returns: + str: A possessive phrase of the form ``"the b of a"``. + """ +if comma_list.__doc__ is None: + comma_list.__doc__ = """Return the items joined by commas. + + Args: + *pargs: Items to join, or a single iterable as the first argument. + **kwargs: Optional ``comma_string`` (default ``', '``) to customize + the separator. + + Returns: + str: The items separated by commas (e.g. + ``comma_list('lions', 'tigers', 'bears')`` returns + ``'lions, tigers, bears'``). + """ +if comma_and_list.__doc__ is None: + comma_and_list.__doc__ = """Return the items joined by commas with "and" before the last item. + + Args: + *pargs: Items to join, or a single iterable as the first argument. + **kwargs: Optional keyword arguments including ``oxford`` (bool, + default ``True``), ``and_string`` (default ``'and'``), + ``comma_string``, ``before_and``, and ``after_and``. + + Returns: + str: An English-language listing such as ``'lions, tigers, and + bears'``. + """ +if add_separators.__doc__ is None: + add_separators.__doc__ = """Return the list items as strings with separators appended. + + Appends ``;`` to all items except the penultimate, which gets + ``'; and'``, and the last, which gets ``'.'``. + + Args: + the_list: The list of items to process. + separator (str, optional): Separator appended to middle items. + Defaults to ``';'``. + last_separator (str, optional): Separator appended to the + penultimate item. Defaults to ``'; and'``. + end_mark (str, optional): Mark appended to the final item. + Defaults to ``'.'``. + + Returns: + list: A list of strings with separators appended. + """ +if nice_number.__doc__ is None: + nice_number.__doc__ = """Return a number expressed as a word for small values, or as a formatted numeral. + + Args: + num: The number to convert. + **kwargs: Optional keyword arguments including ``capitalize`` + (bool), ``language`` (str), and ``use_word`` (bool, default + ``False``). + + Returns: + str: The number as a word (e.g. ``nice_number(4)`` returns + ``'four'``) or as a locale-formatted numeral for larger values. + """ +if quantity_noun.__doc__ is None: + quantity_noun.__doc__ = """Return a number combined with a noun in the appropriate singular or plural form. + + Combines :func:`nice_number` and :func:`noun_plural`. Rounds the number + to the nearest integer unless ``as_integer=False`` is passed. + + Args: + num: The quantity. + noun (str): The singular noun. + **kwargs: Optional keyword arguments including ``as_integer`` + (bool, default ``True``) and other arguments accepted by + :func:`nice_number`. + + Returns: + str: The quantity and noun combined (e.g. ``quantity_noun(2, + 'apple')`` returns ``'two apples'``). + """ +if title_case.__doc__ is None: + title_case.__doc__ = """Return the input string with the first letter of each word capitalized. + + Args: + a (str): The string to convert to title case. + **kwargs: Additional keyword arguments passed to the underlying + language function. + + Returns: + str: The title-cased string (e.g. ``title_case('the importance of + being ernest')`` returns ``'The Importance of Being Ernest'``). + """ + +language_functions.update({ + 'in_the': { + 'en': prefix_constructor('in the ') + }, + 'a_preposition_b': { + 'en': a_preposition_b_default + }, + 'a_in_the_b': { + 'en': middle_constructor(' in the ') + }, + 'i_subjective': { + 'en': lambda *pargs, **kwargs: word('I', **kwargs) + }, + 'he_subjective': { + 'en': lambda *pargs, **kwargs: word('he', **kwargs) + }, + 'she_subjective': { + 'en': lambda *pargs, **kwargs: word('she', **kwargs) + }, + 'genderless_subjective': { + 'en': lambda *pargs, **kwargs: word('they', **kwargs) + }, + 'myself': { + 'en': lambda *pargs, **kwargs: word('myself', **kwargs) + }, + 'itself': { + 'en': lambda *pargs, **kwargs: word('itself', **kwargs) + }, + 'herself': { + 'en': lambda *pargs, **kwargs: word('herself', **kwargs) + }, + 'himself': { + 'en': lambda *pargs, **kwargs: word('himself', **kwargs) + }, + 'themselves': { + 'en': lambda *pargs, **kwargs: word('themselves', **kwargs) + }, + 'genderless_self': { + 'en': lambda *pargs, **kwargs: word('themself', **kwargs) + }, + 'yourself': { + 'en': lambda *pargs, **kwargs: word('yourself', **kwargs) + }, + 'yourselves': { + 'en': lambda *pargs, **kwargs: word('yourselves', **kwargs) + }, + 'ourselves': { + 'en': lambda *pargs, **kwargs: word('ourselves', **kwargs) + }, + 'you_subjective': { + 'en': lambda *pargs, **kwargs: word('you', **kwargs) + }, + 'you_subjective_plural': { + 'en': lambda *pargs, **kwargs: word('you', **kwargs) + }, + 'we_subjective': { + 'en': lambda *pargs, **kwargs: word('we', **kwargs) + }, + 'they_subjective': { + 'en': lambda *pargs, **kwargs: word('they', **kwargs) + }, + 'it_subjective': { + 'en': lambda *pargs, **kwargs: word('it', **kwargs) + }, + 'it_objective': { + 'en': lambda *pargs, **kwargs: word('it', **kwargs) + }, + 'them_objective': { + 'en': lambda *pargs, **kwargs: word('them', **kwargs) + }, + 'genderless_objective': { + 'en': lambda *pargs, **kwargs: word('them', **kwargs) + }, + 'me_objective': { + 'en': lambda *pargs, **kwargs: word('me', **kwargs) + }, + 'him_objective': { + 'en': lambda *pargs, **kwargs: word('him', **kwargs) + }, + 'her_objective': { + 'en': lambda *pargs, **kwargs: word('her', **kwargs) + }, + 'our_objective': { + 'en': lambda *pargs, **kwargs: word('our', **kwargs) + }, + 'you_objective': { + 'en': lambda *pargs, **kwargs: word('you', **kwargs) + }, + 'you_objective_plural': { + 'en': lambda *pargs, **kwargs: word('you', **kwargs) + }, + 'us_objective': { + 'en': lambda *pargs, **kwargs: word('us', **kwargs) + }, + 'are_we': { + 'en': lambda *pargs, **kwargs: word('are we', **kwargs) + }, + 'are_you': { + 'en': lambda *pargs, **kwargs: word('are you', **kwargs) + }, + 'are_you_plural': { + 'en': lambda *pargs, **kwargs: word('are you', **kwargs) + }, + 'am_i': { + 'en': lambda *pargs, **kwargs: word('am I', **kwargs) + }, + 'her': { + 'en': prefix_constructor('her ') + }, + 'his': { + 'en': prefix_constructor('his ') + }, + 'are_word': { + 'en': prefix_constructor('are ') + }, + 'is_word': { + 'en': prefix_constructor('is ') + }, + 'their': { + 'en': prefix_constructor('their ') + }, + 'my_possessive': { + 'en': prefix_constructor('my ') + }, + 'our_possessive': { + 'en': prefix_constructor('our ') + }, + 'of_the': { + 'en': prefix_constructor('of the ') + }, + 'your': { + 'en': prefix_constructor('your ') + }, + 'your_plural': { + 'en': prefix_constructor('your ') + }, + 'some': { + 'en': prefix_constructor('some ') + }, + 'its': { + 'en': prefix_constructor('its ') + }, + 'the': { + 'en': prefix_constructor('the ') + }, + 'these': { + 'en': prefix_constructor('these ') + }, + 'this': { + 'en': prefix_constructor('this ') + }, + 'does_a_b': { + 'en': prefix_constructor_two_arguments('does ') + }, + 'do_a_b': { + 'en': prefix_constructor_two_arguments('do ') + }, + 'did_a_b': { + 'en': prefix_constructor_two_arguments('did ') + }, + 'did_a_b_plural': { + 'en': prefix_constructor_two_arguments('did ') + }, + 'do_i': { + 'en': prefix_constructor('do I ') + }, + 'do_we': { + 'en': prefix_constructor('do we ') + }, + 'do_you': { + 'en': prefix_constructor('do you ') + }, + 'do_you_plural': { + 'en': prefix_constructor('do you ') + }, + 'did_i': { + 'en': prefix_constructor('did I ') + }, + 'did_we': { + 'en': prefix_constructor('did we ') + }, + 'did_you': { + 'en': prefix_constructor('did you ') + }, + 'did_you_plural': { + 'en': prefix_constructor('did you ') + }, + 'was_i': { + 'en': prefix_constructor('was I ') + }, + 'were_we': { + 'en': prefix_constructor('were we ') + }, + 'were_you': { + 'en': prefix_constructor('were you ') + }, + 'were_you_plural': { + 'en': prefix_constructor('were you ') + }, + 'was_a_b': { + 'en': prefix_constructor_two_arguments('was ') + }, + 'were_a_b': { + 'en': prefix_constructor_two_arguments('were ') + }, + 'were_a_b_plural': { + 'en': prefix_constructor_two_arguments('were ') + }, + 'have_i': { + 'en': prefix_constructor('have I ') + }, + 'have_we': { + 'en': prefix_constructor('have we ') + }, + 'have_you': { + 'en': prefix_constructor('have you ') + }, + 'have_you_plural': { + 'en': prefix_constructor('have you ') + }, + 'has_a_b': { + 'en': prefix_constructor_two_arguments('has ') + }, + 'have_a_b': { + 'en': prefix_constructor_two_arguments('have ') + }, + 'verb_past': { + 'en': verb_past_en, + 'es': verb_past_es, + 'de': verb_past_de, + 'fr': verb_past_fr, + 'it': verb_past_it, + 'nl': verb_past_nl + }, + 'verb_present': { + 'en': verb_present_en, + 'es': verb_present_es, + 'de': verb_present_de, + 'fr': verb_present_fr, + 'it': verb_present_it, + 'nl': verb_present_nl + }, + 'noun_plural': { + 'en': noun_plural_en, + 'es': noun_plural_es, + 'de': noun_plural_de, + 'fr': noun_plural_fr, + 'it': noun_plural_it, + 'nl': noun_plural_nl + }, + 'noun_singular': { + 'en': noun_singular_en, + 'es': noun_singular_es, + 'de': noun_singular_de, + 'fr': noun_singular_fr, + 'it': noun_singular_it, + 'nl': noun_singular_nl + }, + 'indefinite_article': { + 'en': indefinite_article_en, + 'es': indefinite_article_es, + 'de': indefinite_article_de, + 'it': indefinite_article_it + }, + 'period_list': { + '*': lambda: [[12, word("Per Month")], [1, word("Per Year")], [52, word("Per Week")], [24, word("Twice Per Month")], [26, word("Every Two Weeks")]] + }, + 'name_suffix': { + '*': lambda: ['Jr', 'Sr', 'II', 'III', 'IV', 'V', 'VI'] + }, + 'currency': { + '*': currency_default + }, + 'possessify': { + 'en': possessify_en + }, + 'possessify_long': { + 'en': double_prefix_constructor_reverse('the ', ' of the ') + }, + 'comma_and_list': { + 'en': comma_and_list_en, + 'es': comma_and_list_es, + 'de': comma_and_list_de + }, + 'comma_list': { + 'en': comma_list_en + }, + 'add_separators': { + 'en': add_separators_en + }, + 'nice_number': { + '*': nice_number_default + }, + 'quantity_noun': { + '*': quantity_noun_default + }, + 'capitalize': { + '*': capitalize_default + }, + 'title_case': { + '*': titlecasestr + }, + 'salutation': { + '*': salutation_default + } +}) diff --git a/docassemble_base/docassemble/base/language/language_de.py b/docassemble_base/docassemble/base/language/language_de.py new file mode 100644 index 000000000..d7b41be2a --- /dev/null +++ b/docassemble_base/docassemble/base/language/language_de.py @@ -0,0 +1,74 @@ +from docassemble.base.pattern import pattern_de +from .core import ensure_definition +from .capitalization import capitalize +from .numbers import number_or_length +from .language_en import comma_and_list_en + +def comma_and_list_de(*pargs, **kwargs): + if 'and_string' not in kwargs: + kwargs['and_string'] = 'und' + if 'oxford' not in kwargs: + kwargs['oxford'] = False + return comma_and_list_en(*pargs, **kwargs) + + +def verb_present_de(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + new_args = [] + for arg in pargs: + new_args.append(str(arg)) + if len(new_args) < 2: + new_args.append('3sg') + if new_args[1] == 'pl': + new_args[1] = '3pl' + output = pattern_de.conjugate(*new_args, **kwargs) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def verb_past_de(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + new_args = [] + for arg in pargs: + new_args.append(arg) + if len(new_args) < 2: + new_args.append('3sgp') + if new_args[1] == 'ppl': + new_args[1] = '3ppl' + output = pattern_de.conjugate(*new_args, **kwargs) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def noun_plural_de(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + if kwargs.get('noun_is_singular', False): + noun = pargs[0] + else: + noun = noun_singular_de(pargs[0]) + if len(pargs) >= 2 and number_or_length(pargs[1]) == 1: + return str(noun) + output = pattern_de.pluralize(str(noun)) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def noun_singular_de(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + if len(pargs) >= 2 and number_or_length(pargs[1]) != 1: + return pargs[0] + output = pattern_de.singularize(str(pargs[0])) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def indefinite_article_de(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + output = pattern_de.article(str(pargs[0]).lower()) + " " + str(pargs[0]) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output diff --git a/docassemble_base/docassemble/base/language/language_en.py b/docassemble_base/docassemble/base/language/language_en.py new file mode 100644 index 000000000..f766d0bf3 --- /dev/null +++ b/docassemble_base/docassemble/base/language/language_en.py @@ -0,0 +1,162 @@ +from collections.abc import Iterable +from docassemble.base.pattern import pattern_en +from docassemble.base.thread_context import this_thread +from docassemble.base.language.utils import fix_punctuation +from .core import ensure_definition +from .capitalization import capitalize +from .numbers import number_or_length +from .words import word + +def noun_plural_en(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + if kwargs.get('noun_is_singular', False): + noun = pargs[0] + else: + noun = noun_singular_en(pargs[0]) + if len(pargs) >= 2 and number_or_length(pargs[1]) == 1: + return str(noun) + output = pattern_en.pluralize(str(noun)) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def noun_singular_en(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + if len(pargs) >= 2 and number_or_length(pargs[1]) != 1: + return pargs[0] + output = pattern_en.singularize(str(pargs[0])) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def indefinite_article_en(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + output = pattern_en.article(str(pargs[0]).lower()) + " " + str(pargs[0]) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + +def verb_present_en(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + new_args = [] + for arg in pargs: + new_args.append(str(arg)) + if len(new_args) < 2: + new_args.append('3sg') + output = pattern_en.conjugate(*new_args, **kwargs) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def verb_past_en(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + new_args = [] + for arg in pargs: + new_args.append(arg) + if len(new_args) < 2: + new_args.append('3sgp') + output = pattern_en.conjugate(*new_args, **kwargs) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def possessify_en(a, b, **kwargs): + ensure_definition(a, b, **kwargs) + if this_thread.evaluation_context == 'docx': + apostrophe = "’" + else: + apostrophe = "'" + if 'plural' in kwargs and kwargs['plural']: + middle = apostrophe + " " + else: + middle = apostrophe + "s " + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(str(a)) + str(middle) + str(b) + return str(a) + str(middle) + str(b) + + +def comma_list_en(*pargs, **kwargs): + """Returns the arguments separated by commas. If the first argument is a list, + that list is used. Otherwise, the arguments are treated as individual items. + See also comma_and_list().""" + ensure_definition(*pargs, **kwargs) + comma_string = kwargs.get('comma_string', ', ') + the_list = [] + for parg in pargs: + if isinstance(parg, str): + the_list.append(parg) + elif (hasattr(parg, 'instanceName') and hasattr(parg, 'elements')) or isinstance(parg, Iterable): + for sub_parg in parg: + the_list.append(str(sub_parg)) + else: + the_list.append(str(parg)) + return comma_string.join(the_list) + + +def comma_and_list_en(*pargs, **kwargs): + """Returns an English-language listing of the arguments. If the first argument is a list, + that list is used. Otherwise, the arguments are treated as individual items in the list. + Use the optional argument oxford=False if you do not want a comma before the "and." + See also comma_list().""" + ensure_definition(*pargs, **kwargs) + and_string = kwargs.get('and_string', word('and')) + comma_string = kwargs.get('comma_string', ', ') + if 'oxford' in kwargs and kwargs['oxford'] is False: + extracomma = "" + else: + extracomma = comma_string.strip() + before_and = kwargs.get('before_and', ' ') + after_and = kwargs.get('after_and', ' ') + the_list = [] + for parg in pargs: + if isinstance(parg, str): + the_list.append(parg) + elif (hasattr(parg, 'instanceName') and hasattr(parg, 'elements')) or isinstance(parg, Iterable): + for sub_parg in parg: + the_list.append(str(sub_parg)) + else: + the_list.append(str(parg)) + if len(the_list) == 0: + return str('') + if len(the_list) == 1: + return the_list[0] + if len(the_list) == 2: + return the_list[0] + before_and + and_string + after_and + the_list[1] + return comma_string.join(the_list[:-1]) + extracomma + before_and + and_string + after_and + the_list[-1] + + +def add_separators_en(*pargs, **kwargs): + """Accepts a list and returns a list, with semicolons after each item, + except "and" after the penultimate item and a period after the + last. + + """ + ensure_definition(*pargs, **kwargs) + separator = kwargs.get('separator', ';') + last_separator = kwargs.get('last_separator', '; ' + word("and")) + end_mark = kwargs.get('end_mark', '.') + the_list = [] + for parg in pargs: + if isinstance(parg, str): + the_list.append(parg.rstrip()) + elif (hasattr(parg, 'instanceName') and hasattr(parg, 'elements')) or isinstance(parg, Iterable): + for sub_parg in parg: + the_list.append(str(sub_parg).rstrip()) + else: + the_list.append(str(parg).rstrip()) + if len(the_list) == 0: + return the_list + if len(the_list) == 1: + return [fix_punctuation(the_list[0], mark=end_mark)] + for indexno in range(len(the_list) - 2): # for 4: 0, 1; for 3: 0; for 2: [] + the_list[indexno] = the_list[indexno].rstrip(',') + the_list[indexno] = fix_punctuation(the_list[indexno], mark=separator) + if not the_list[-2].endswith(last_separator): + the_list[-2] = the_list[-2].rstrip(last_separator[0]) + the_list[-2] += last_separator + the_list[-1] = fix_punctuation(the_list[-1], mark=end_mark) + return the_list diff --git a/docassemble_base/docassemble/base/language/language_es.py b/docassemble_base/docassemble/base/language/language_es.py new file mode 100644 index 000000000..30d4fb873 --- /dev/null +++ b/docassemble_base/docassemble/base/language/language_es.py @@ -0,0 +1,72 @@ +from docassemble.base.pattern import pattern_es +from .core import ensure_definition +from .capitalization import capitalize +from .numbers import number_or_length +from .language_en import comma_and_list_en + +def comma_and_list_es(*pargs, **kwargs): + if 'and_string' not in kwargs: + kwargs['and_string'] = 'y' + return comma_and_list_en(*pargs, **kwargs) + + +def verb_present_es(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + new_args = [] + for arg in pargs: + new_args.append(str(arg)) + if len(new_args) < 2: + new_args.append('3sg') + if new_args[1] == 'pl': + new_args[1] = '3pl' + output = pattern_es.conjugate(*new_args, **kwargs) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def verb_past_es(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + new_args = [] + for arg in pargs: + new_args.append(arg) + if len(new_args) < 2: + new_args.append('3sgp') + if new_args[1] == 'ppl': + new_args[1] = '3ppl' + output = pattern_es.conjugate(*new_args, **kwargs) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def noun_plural_es(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + if kwargs.get('noun_is_singular', False): + noun = pargs[0] + else: + noun = noun_singular_es(pargs[0]) + if len(pargs) >= 2 and number_or_length(pargs[1]) == 1: + return str(noun) + output = pattern_es.pluralize(str(noun)) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def noun_singular_es(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + if len(pargs) >= 2 and number_or_length(pargs[1]) != 1: + return pargs[0] + output = pattern_es.singularize(str(pargs[0])) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def indefinite_article_es(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + output = pattern_es.article(str(pargs[0]).lower()) + " " + str(pargs[0]) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output diff --git a/docassemble_base/docassemble/base/language/language_fr.py b/docassemble_base/docassemble/base/language/language_fr.py new file mode 100644 index 000000000..4af6457eb --- /dev/null +++ b/docassemble_base/docassemble/base/language/language_fr.py @@ -0,0 +1,65 @@ +from docassemble.base.pattern import pattern_fr +from .core import ensure_definition +from .capitalization import capitalize +from .numbers import number_or_length + +def verb_present_fr(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + new_args = [] + for arg in pargs: + new_args.append(str(arg)) + if len(new_args) < 2: + new_args.append('3sg') + if new_args[1] == 'pl': + new_args[1] = '3pl' + output = pattern_fr.conjugate(*new_args, **kwargs) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def verb_past_fr(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + new_args = [] + for arg in pargs: + new_args.append(arg) + if len(new_args) < 2: + new_args.append('3sgp') + if new_args[1] == 'ppl': + new_args[1] = '3ppl' + output = pattern_fr.conjugate(*new_args, **kwargs) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def noun_plural_fr(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + if kwargs.get('noun_is_singular', False): + noun = pargs[0] + else: + noun = noun_singular_fr(pargs[0]) + if len(pargs) >= 2 and number_or_length(pargs[1]) == 1: + return str(noun) + output = pattern_fr.pluralize(str(noun)) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def noun_singular_fr(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + if len(pargs) >= 2 and number_or_length(pargs[1]) != 1: + return pargs[0] + output = pattern_fr.singularize(str(pargs[0])) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def indefinite_article_fr(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + output = pattern_fr.article(str(pargs[0]).lower()) + " " + str(pargs[0]) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output diff --git a/docassemble_base/docassemble/base/language/language_it.py b/docassemble_base/docassemble/base/language/language_it.py new file mode 100644 index 000000000..c517fa844 --- /dev/null +++ b/docassemble_base/docassemble/base/language/language_it.py @@ -0,0 +1,65 @@ +from docassemble.base.pattern import pattern_it +from .core import ensure_definition +from .capitalization import capitalize +from .numbers import number_or_length + +def verb_present_it(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + new_args = [] + for arg in pargs: + new_args.append(str(arg)) + if len(new_args) < 2: + new_args.append('3sg') + if new_args[1] == 'pl': + new_args[1] = '3pl' + output = pattern_it.conjugate(*new_args, **kwargs) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def verb_past_it(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + new_args = [] + for arg in pargs: + new_args.append(arg) + if len(new_args) < 2: + new_args.append('3sgp') + if new_args[1] == 'ppl': + new_args[1] = '3ppl' + output = pattern_it.conjugate(*new_args, **kwargs) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def noun_plural_it(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + if kwargs.get('noun_is_singular', False): + noun = pargs[0] + else: + noun = noun_singular_it(pargs[0]) + if len(pargs) >= 2 and number_or_length(pargs[1]) == 1: + return str(noun) + output = pattern_it.pluralize(str(noun)) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def noun_singular_it(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + if len(pargs) >= 2 and number_or_length(pargs[1]) != 1: + return pargs[0] + output = pattern_it.singularize(str(pargs[0])) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def indefinite_article_it(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + output = pattern_it.article(str(pargs[0]).lower()) + " " + str(pargs[0]) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output diff --git a/docassemble_base/docassemble/base/language/language_nl.py b/docassemble_base/docassemble/base/language/language_nl.py new file mode 100644 index 000000000..6671eb3e9 --- /dev/null +++ b/docassemble_base/docassemble/base/language/language_nl.py @@ -0,0 +1,65 @@ +from docassemble.base.pattern import pattern_nl +from .core import ensure_definition +from .capitalization import capitalize +from .numbers import number_or_length + +def verb_present_nl(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + new_args = [] + for arg in pargs: + new_args.append(str(arg)) + if len(new_args) < 2: + new_args.append('3sg') + if new_args[1] == 'pl': + new_args[1] = '3pl' + output = pattern_nl.conjugate(*new_args, **kwargs) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def verb_past_nl(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + new_args = [] + for arg in pargs: + new_args.append(arg) + if len(new_args) < 2: + new_args.append('3sgp') + if new_args[1] == 'ppl': + new_args[1] = '3ppl' + output = pattern_nl.conjugate(*new_args, **kwargs) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def noun_plural_nl(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + if kwargs.get('noun_is_singular', False): + noun = pargs[0] + else: + noun = noun_singular_nl(pargs[0]) + if len(pargs) >= 2 and number_or_length(pargs[1]) == 1: + return str(noun) + output = pattern_nl.pluralize(str(noun)) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def noun_singular_nl(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + if len(pargs) >= 2 and number_or_length(pargs[1]) != 1: + return pargs[0] + output = pattern_nl.singularize(str(pargs[0])) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output + + +def indefinite_article_nl(*pargs, **kwargs): + ensure_definition(*pargs, **kwargs) + output = pattern_nl.article(str(pargs[0]).lower()) + " " + str(pargs[0]) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(output) + return output diff --git a/docassemble_base/docassemble/base/language/numbers.py b/docassemble_base/docassemble/base/language/numbers.py new file mode 100644 index 000000000..9ea728067 --- /dev/null +++ b/docassemble_base/docassemble/base/language/numbers.py @@ -0,0 +1,203 @@ +import decimal +import locale +from typing import Callable +import num2words +from docassemble.base.thread_context import this_thread +from .capitalization import capitalize +from .control import get_language +from .core import ( + language_function_constructor, + ensure_definition, + update_language_function, +) + +ordinal_functions: dict[str, Callable] = {} + +ordinal_numbers: dict[str, dict[str, str]] = {} + +nice_numbers: dict[str, dict[str, str]] = {} + +ordinal_number = language_function_constructor('ordinal_number') +ordinal = language_function_constructor('ordinal') +nice_number = language_function_constructor('nice_number') + +if ordinal_number.__doc__ is None: + ordinal_number.__doc__ = """Return the ordinal form of a cardinal number. + + Args: + num: The cardinal number (1-based). + **kwargs: Optional keyword arguments including ``capitalize`` + (bool) and ``use_word`` (bool, default depends on the value). + + Returns: + str: The ordinal form (e.g. ``ordinal_number(8)`` returns + ``'eighth'``; ``ordinal_number(8, use_word=False)`` returns + ``'8th'``). + """ +if ordinal.__doc__ is None: + ordinal.__doc__ = """Return the ordinal form of a zero-based index. + + Equivalent to ``ordinal_number(num + 1)``. This is useful when working + with zero-based list indexes. + + Args: + num: The zero-based index. + **kwargs: Optional keyword arguments passed to :func:`ordinal_number`. + + Returns: + str: The ordinal form (e.g. ``ordinal(0)`` returns ``'first'``; + ``ordinal(22)`` returns ``'23rd'``). + """ + +def string_to_number(number): + try: + float_number = float(number) + int_number = int(number) + if float_number == int_number: + return int_number + return float_number + except: + return number + + +def number_to_word(number, **kwargs): + language = kwargs.get('language', None) + capitalize_arg = kwargs.get('capitalize', False) + function = kwargs.get('function', None) + raise_on_error = kwargs.get('raise_on_error', False) + if function not in ('ordinal', 'ordinal_num'): + function = 'cardinal' + if language is None: + language = get_language() + for lang, loc in (('en', 'en_GB'), ('en', 'en_IN'), ('es', 'es_CO'), ('es', 'es_VE'), ('fr', 'fr_CH'), ('fr', 'fr_BE'), ('fr', 'fr_DZ'), ('pt', 'pt_BR')): + if language == lang and this_thread.locale.startswith(loc): + language = loc + break + number = string_to_number(number) + if raise_on_error: + the_word = num2words.num2words(number, lang=language, to=function) + else: + try: + the_word = num2words.num2words(number, lang=language, to=function) + except NotImplementedError: + the_word = str(number) + if capitalize_arg: + return capitalize(the_word) + return the_word + + +def ordinal_default(the_number, **kwargs): + """Returns the "first," "second," "third," etc. for a given number, which is expected to + be an index starting with zero. ordinal(0) returns "first." For a more literal ordinal + number function, see ordinal_number().""" + result = ordinal_number(int(float(the_number)) + 1, **kwargs) + if 'capitalize' in kwargs and kwargs['capitalize']: + return capitalize(result) + return result + + +def nice_number_default(the_number, **kwargs): + """Returns the number as a word in the current language.""" + capitalize_arg = kwargs.get('capitalize', False) + language = kwargs.get('language', None) + use_word = kwargs.get('use_word', None) + ensure_definition(the_number, capitalize_arg, language) + if language is None: + language = this_thread.language + if language in nice_numbers: + language_to_use = language + elif '*' in nice_numbers: + language_to_use = '*' + else: + language_to_use = 'en' + if isinstance(the_number, float): + the_number = float(decimal.Context(prec=8).create_decimal_from_float(the_number)) + if int(float(the_number)) == float(the_number): + the_number = int(float(the_number)) + is_integer = True + else: + is_integer = False + if language_to_use in nice_numbers and str(the_number) in nice_numbers[language_to_use]: + the_word = nice_numbers[language_to_use][str(the_number)] + if capitalize_arg: + return capitalize(the_word) + return the_word + if use_word or (is_integer and 0 <= the_number < 11 and use_word is not False): + try: + return number_to_word(the_number, **kwargs) + except: + pass + if isinstance(the_number, int): + return str(locale.format_string("%d", the_number, grouping=True)) + return str(locale.format_string("%.2f", float(the_number), grouping=True)).rstrip('0') + + +def ordinal_function_en(i, **kwargs): + try: + i = int(i) + except: + i = 0 + use_word = kwargs.get('use_word', None) + if use_word is True: + kwargs['function'] = 'ordinal' + elif use_word is False: + kwargs['function'] = 'ordinal_num' + else: + if i < 11: + kwargs['function'] = 'ordinal' + else: + kwargs['function'] = 'ordinal_num' + return number_to_word(i, **kwargs) + + +def ordinal_number_default(the_number, **kwargs): + """Returns the "first," "second," "third," etc. for a given number. + ordinal_number(1) returns "first." For a function that can be used + on index numbers that start with zero, see ordinal().""" + num = str(the_number) + if kwargs.get('use_word', True): + if this_thread.language in ordinal_numbers and num in ordinal_numbers[this_thread.language]: + return ordinal_numbers[this_thread.language][num] + if '*' in ordinal_numbers and num in ordinal_numbers['*']: + return ordinal_numbers['*'][num] + if this_thread.language in ordinal_functions: + language_to_use = this_thread.language + elif '*' in ordinal_functions: + language_to_use = '*' + else: + language_to_use = 'en' + return ordinal_functions[language_to_use](the_number, **kwargs) + + +def update_nice_numbers(lang, defs): + if lang not in nice_numbers: + nice_numbers[lang] = {} + for number, the_word in defs.items(): + nice_numbers[lang][str(number)] = the_word + + +def update_ordinal_numbers(lang, defs): + if lang not in ordinal_numbers: + ordinal_numbers[lang] = {} + for number, the_word in defs.items(): + ordinal_numbers[lang][str(number)] = the_word + + +def update_ordinal_function(lang, func): + ordinal_functions[lang] = func + + +def number_or_length(target): + if isinstance(target, (int, float)): + return target + if isinstance(target, (list, dict, set, tuple)) or (hasattr(target, 'elements') and isinstance(target.elements, (list, dict, set))): + return len(target) + if target: + return 2 + return 1 + + +update_ordinal_function('en', ordinal_function_en) +update_ordinal_function('*', ordinal_function_en) +update_language_function('*', 'ordinal_number', ordinal_number_default) +update_language_function('*', 'ordinal', ordinal_default) diff --git a/docassemble_base/docassemble/base/language/utils.py b/docassemble_base/docassemble/base/language/utils.py new file mode 100644 index 000000000..3f8501e77 --- /dev/null +++ b/docassemble_base/docassemble/base/language/utils.py @@ -0,0 +1,29 @@ +from docassemble.base.language.core import ensure_definition + +def fix_punctuation(text, mark=None, other_marks=None): + """Ensure the text ends with a punctuation mark, adding one if necessary. + + Args: + text (str): The text to check. + mark (str, optional): The punctuation mark to append if none is + present. Defaults to ``'.'``. + other_marks (list, optional): A list of punctuation marks that are + considered acceptable endings. Defaults to ``['.', '?', '!']``. + + Returns: + str: The text, possibly with a punctuation mark appended. + """ + ensure_definition(text, mark, other_marks) + if other_marks is None: + other_marks = ['.', '?', '!'] + if not isinstance(other_marks, list): + other_marks = list(other_marks) + if mark is None: + mark = '.' + text = text.rstrip() + if mark == '': + return text + for end_mark in set([mark] + other_marks): + if text.endswith(end_mark): + return text + return text + mark diff --git a/docassemble_base/docassemble/base/language/words.py b/docassemble_base/docassemble/base/language/words.py new file mode 100644 index 000000000..e62e99e87 --- /dev/null +++ b/docassemble_base/docassemble/base/language/words.py @@ -0,0 +1,75 @@ +from docassemble.base.thread_context import this_thread +from .capitalization import capitalize + +word_collection = { + 'en': { + 'This field is required.': 'You need to fill this in.', + "Country Code": 'Country Code (e.g., "us")', + "First Subdivision": 'State Abbreviation (e.g., "NY")', + "Second Subdivision": "County", + "Third Subdivision": "Municipality", + } +} + + +def words(): + return word_collection[this_thread.language] + + +class LazyWord: + + def __init__(self, *args, **kwargs): + if len(kwargs) > 0: + self.original = args[0] % kwargs + else: + self.original = args[0] + + def __mod__(self, other): + return word(self.original) % other + + def __str__(self): + return word(self.original) + + +def word(the_word, **kwargs): + """Return the word translated into the current language. + + If no translation is found for the current language, the input is + returned unchanged. Used throughout docassemble to support + multilingual interviews. + + Args: + the_word (str): The word or phrase to translate. + **kwargs: Optional keyword arguments. Pass ``language`` to + look up a translation for a specific language, or + ``capitalize=True`` to capitalize the result. + + Returns: + str: The translated (or original) word. + """ + # Currently, no kwargs are used, but in the future, this function could be + # expanded to use kwargs. For example, for languages with gendered words, + # the gender could be passed as a keyword argument. + if the_word is True: + the_word = 'yes' + elif the_word is False: + the_word = 'no' + elif the_word is None: + the_word = "I don't know" + if isinstance(the_word, LazyWord): + the_word = the_word.original + try: + the_word = word_collection[kwargs.get('language', this_thread.language)][the_word] + except: + the_word = str(the_word) + if kwargs.get('capitalize', False): + return capitalize(the_word) + return the_word + + +def update_word_collection(lang, defs): + if lang not in word_collection: + word_collection[lang] = {} + for the_word, translation in defs.items(): + if translation is not None: + word_collection[lang][the_word] = translation diff --git a/docassemble_base/docassemble/base/legal.py b/docassemble_base/docassemble/base/legal.py index 7e3a9317e..d2f07ad66 100644 --- a/docassemble_base/docassemble/base/legal.py +++ b/docassemble_base/docassemble/base/legal.py @@ -1,7 +1,278 @@ +# ruff: noqa: F401 +# pylint: disable=unused-import from itertools import chain -from docassemble.base.functions import alpha, roman, item_label, comma_and_list, get_language, set_language, get_dialect, get_voice, set_country, get_country, word, comma_list, ordinal, ordinal_number, need, nice_number, quantity_noun, possessify, verb_past, verb_present, noun_plural, noun_singular, space_to_underscore, force_ask, force_gather, period_list, name_suffix, currency, currency_symbol, indefinite_article, nodoublequote, capitalize, title_case, url_of, do_you, did_you, does_a_b, did_a_b, your, her, his, is_word, get_locale, set_locale, update_locale, process_action, url_action, get_info, set_info, get_config, prevent_going_back, qr_code, action_menu_item, from_b64_json, defined, define, value, message, response, json_response, command, single_paragraph, quote_paragraphs, location_returned, location_known, user_lat_lon, interview_url, interview_url_action, interview_url_as_qr, interview_url_action_as_qr, interview_email, get_emails, get_default_timezone, user_logged_in, interface, user_privileges, user_has_privilege, user_info, current_context, action_arguments, action_argument, background_action, background_response, background_response_action, background_error_action, us, set_live_help_status, chat_partners_available, phone_number_in_e164, phone_number_formatted, phone_number_is_valid, countries_list, country_name, write_record, read_records, delete_record, variables_as_json, all_variables, language_from_browser, device, plain, bold, italic, states_list, state_name, subdivision_type, indent, raw, fix_punctuation, set_progress, get_progress, referring_url, undefine, invalidate, dispatch, yesno, noyes, split, showif, showifdef, phone_number_part, set_parts, log, encode_name, decode_name, interview_list, interview_menu, server_capabilities, session_tags, get_chat_log, get_user_list, get_user_info, set_user_info, get_user_secret, create_user, invite_user, create_session, get_session_variables, set_session_variables, get_question_data, go_back_in_session, manage_privileges, redact, forget_result_of, re_run_logic, reconsider, set_title, set_save_status, single_to_double_newlines, verbatim, add_separators, store_variables_snapshot, update_terms, set_variables, language_name, run_action_in_session, static_image # noqa: F401 # pylint: disable=unused-import -from docassemble.base.util import LatitudeLongitude, RoleChangeTracker, Name, IndividualName, Address, City, Event, Person, Thing, Individual, ChildList, FinancialList, PeriodicFinancialList, Income, Asset, Expense, Value, PeriodicValue, OfficeList, Organization, send_email, send_sms, send_fax, map_of, last_access_time, last_access_delta, last_access_days, last_access_hours, last_access_minutes, returning_user, timezone_list, as_datetime, current_datetime, date_difference, date_interval, today, month_of, day_of, dow_of, year_of, format_date, format_datetime, format_time, DARedis, DACloudStorage, DAGoogleAPI, SimpleTextMachineLearner, MachineLearningEntry, RandomForestMachineLearner, SVMMachineLearner, ocr_file, ocr_file_in_background, read_qr, get_sms_session, initiate_sms_session, terminate_sms_session, path_and_mimetype, run_python_module, pdf_concatenate, include_docx_template, start_time, zip_file, validation_error, DAValidationError, action_button_html, url_ask, overlay_pdf, DAStore, explain, clear_explanations, logic_explanation, set_status, get_status, DAWeb, DAWebError, json, re, iso_country, assemble_docx, docx_concatenate, task_performed, task_not_yet_performed, mark_task_as_performed, times_task_performed, set_task_counter, stash_data, retrieve_stashed_data, DABreadCrumbs, DAOAuth, DAObject, DAList, DADict, DAOrderedDict, DASet, DAFile, DAFileCollection, DAFileList, DAStaticFile, DAEmail, DAEmailRecipient, DAEmailRecipientList, DATemplate, DAEmpty, DALink, selections, objects_from_file, RelationshipTree, DAContext, DA, DAGlobal, transform_json_variables # noqa: F401 # pylint: disable=unused-import -# from docassemble.base.logger import logmessage +from docassemble.base.functions import ( + alpha, + roman, + item_label, + comma_and_list, + get_language, + set_language, + get_dialect, + get_voice, + set_country, + get_country, + word, + comma_list, + ordinal, + ordinal_number, + need, + nice_number, + quantity_noun, + possessify, + verb_past, + verb_present, + noun_plural, + noun_singular, + space_to_underscore, + force_ask, + force_gather, + period_list, + name_suffix, + currency, + currency_symbol, + indefinite_article, + nodoublequote, + capitalize, + title_case, + url_of, + do_you, + did_you, + does_a_b, + did_a_b, + your, + her, + his, + is_word, + get_locale, + set_locale, + update_locale, + process_action, + url_action, + get_info, + set_info, + get_config, + prevent_going_back, + qr_code, + action_menu_item, + from_b64_json, + defined, + define, + value, + message, + response, + json_response, + command, + single_paragraph, + quote_paragraphs, + location_returned, + location_known, + user_lat_lon, + interview_url, + interview_url_action, + interview_url_as_qr, + interview_url_action_as_qr, + interview_email, + get_emails, + get_default_timezone, + user_logged_in, + interface, + user_privileges, + user_has_privilege, + user_info, + current_context, + action_arguments, + action_argument, + background_action, + background_response, + background_response_action, + background_error_action, + us, + set_live_help_status, + chat_partners_available, + phone_number_in_e164, + phone_number_formatted, + phone_number_is_valid, + countries_list, + country_name, + write_record, + read_records, + delete_record, + variables_as_json, + all_variables, + language_from_browser, + device, + plain, + bold, + italic, + states_list, + state_name, + subdivision_type, + indent, + raw, + fix_punctuation, + set_progress, + get_progress, + referring_url, + undefine, + invalidate, + dispatch, + yesno, + noyes, + split, + showif, + showifdef, + phone_number_part, + set_parts, + log, + encode_name, + decode_name, + interview_list, + interview_menu, + server_capabilities, + session_tags, + get_chat_log, + get_user_list, + get_user_info, + set_user_info, + get_user_secret, + create_user, + invite_user, + create_session, + get_session_variables, + set_session_variables, + get_question_data, + go_back_in_session, + manage_privileges, + redact, + forget_result_of, + re_run_logic, + reconsider, + set_title, + set_save_status, + single_to_double_newlines, + verbatim, + add_separators, + store_variables_snapshot, + update_terms, + set_variables, + language_name, + run_action_in_session, + static_image, +) +from docassemble.base.util import ( + LatitudeLongitude, + RoleChangeTracker, + Name, + IndividualName, + Address, + City, + Event, + Person, + Thing, + Individual, + ChildList, + FinancialList, + PeriodicFinancialList, + Income, + Asset, + Expense, + Value, + PeriodicValue, + OfficeList, + Organization, + send_email, + send_sms, + send_fax, + map_of, + last_access_time, + last_access_delta, + last_access_days, + last_access_hours, + last_access_minutes, + returning_user, + timezone_list, + as_datetime, + current_datetime, + date_difference, + date_interval, + today, + month_of, + day_of, + dow_of, + year_of, + format_date, + format_datetime, + format_time, + DARedis, + DACloudStorage, + DAGoogleAPI, + SimpleTextMachineLearner, + MachineLearningEntry, + RandomForestMachineLearner, + SVMMachineLearner, + ocr_file, + ocr_file_in_background, + read_qr, + get_sms_session, + initiate_sms_session, + terminate_sms_session, + path_and_mimetype, + run_python_module, + pdf_concatenate, + include_docx_template, + start_time, + zip_file, + validation_error, + DAValidationError, + action_button_html, + url_ask, + overlay_pdf, + DAStore, + explain, + clear_explanations, + logic_explanation, + set_status, + get_status, + DAWeb, + DAWebError, + json, + re, + iso_country, + assemble_docx, + docx_concatenate, + task_performed, + task_not_yet_performed, + mark_task_as_performed, + times_task_performed, + set_task_counter, + stash_data, + retrieve_stashed_data, + DABreadCrumbs, + DAOAuth, + DAObject, + DAList, + DADict, + DAOrderedDict, + DASet, + DAFile, + DAFileCollection, + DAFileList, + DAStaticFile, + DAEmail, + DAEmailRecipient, + DAEmailRecipientList, + DATemplate, + DAEmpty, + DALink, + selections, + objects_from_file, + RelationshipTree, + DAContext, + DA, + DAGlobal, + transform_json_variables, +) __all__ = [ 'alpha', @@ -353,8 +624,8 @@ def init(self, *pargs, **kwargs): self.initializeAttribute('court', self.CourtClass) self.initializeAttribute('defendant', self.PartyListClass) self.initializeAttribute('plaintiff', self.PartyListClass) - self.firstParty = self.plaintiff - self.secondParty = self.defendant + self.firstParty = self.plaintiff # pylint: disable=invalid-name + self.secondParty = self.defendant # pylint: disable=invalid-name self.is_solo_action = False self.state = None self.action_type = 'plaintiff defendant' diff --git a/docassemble_base/docassemble/base/mako/ast.py b/docassemble_base/docassemble/base/mako/ast.py index e8fec80a8..fb6c0a41c 100644 --- a/docassemble_base/docassemble/base/mako/ast.py +++ b/docassemble_base/docassemble/base/mako/ast.py @@ -11,7 +11,7 @@ from docassemble.base.mako import exceptions from docassemble.base.mako import pyparser -from docassemble.base.astparser import myvisitnode +from docassemble.base.astparser import MyVisitNode from docassemble.base.astparser import ast as base_ast @@ -21,7 +21,7 @@ class PythonCode: def __init__(self, code, **exception_kwargs): if isinstance(code, str): - myvisitor = myvisitnode() + myvisitor = MyVisitNode() t = base_ast.parse(code.strip()) myvisitor.visit(t) self.names_used = set() diff --git a/docassemble_base/docassemble/base/mako/exceptions.py b/docassemble_base/docassemble/base/mako/exceptions.py index c353f4945..fbe2e9154 100644 --- a/docassemble_base/docassemble/base/mako/exceptions.py +++ b/docassemble_base/docassemble/base/mako/exceptions.py @@ -161,7 +161,7 @@ def _init(self, trcback): (line_map, template_lines, template_filename) = mods[filename] except KeyError: try: - info = mako.template._get_module_info(filename) + info = docassemble.base.mako.template._get_module_info(filename) module_source = info.code template_source = info.source template_filename = ( @@ -185,7 +185,7 @@ def _init(self, trcback): template_ln = 1 - mtm = mako.template.ModuleInfo + mtm = docassemble.base.mako.template.ModuleInfo source_map = mtm.get_module_source_metadata( module_source, full_line_map=True ) diff --git a/docassemble_base/docassemble/base/microsoft.py b/docassemble_base/docassemble/base/microsoft.py index 1e8a43fe5..6f0d8cf4c 100644 --- a/docassemble_base/docassemble/base/microsoft.py +++ b/docassemble_base/docassemble/base/microsoft.py @@ -5,7 +5,12 @@ import re import logging import yaml -from azure.storage.blob import BlobServiceClient, BlobSasPermissions, ContentSettings, generate_blob_sas +from azure.storage.blob import ( + BlobServiceClient, + BlobSasPermissions, + ContentSettings, + generate_blob_sas, +) from azure.identity import ManagedIdentityCredential from azure.keyvault.secrets import SecretClient from docassemble.base.error import DAException @@ -16,7 +21,7 @@ epoch = datetime.datetime(1970, 1, 1, 0, 0, tzinfo=datetime.timezone.utc) -class azureobject: +class AzureObject: def __init__(self, azure_config): if ('key vault name' in azure_config and azure_config['key vault name'] is not None and 'managed identity' in azure_config and azure_config['managed identity'] is not None): @@ -43,7 +48,7 @@ def __init__(self, azure_config): raise DAException("Cannot connect to Azure without account name, account key, and container specified") def get_key(self, key_name): - new_key = azurekey(self, key_name, load=False) + new_key = AzureKey(self, key_name, load=False) if new_key.exists(): new_key.get_properties() new_key.does_exist = True @@ -54,17 +59,17 @@ def get_key(self, key_name): def search_key(self, key_name): for blob in self.container_client.list_blobs(name_starts_with=key_name): if blob.name == key_name: - return azurekey(self, blob.name) + return AzureKey(self, blob.name) return None def list_keys(self, prefix): output = [] for blob in self.container_client.list_blobs(name_starts_with=prefix): - output.append(azurekey(self, blob.name)) + output.append(AzureKey(self, blob.name)) return output def get_secret(self, key_vault_reference): - new_secret = azuresecret(self, key_vault_reference) + new_secret = AzureSecret(self, key_vault_reference) return new_secret.get_secret_as_string() def replace_secrets(self, match): @@ -78,7 +83,7 @@ def load_with_secrets(self, config): return loaded_config_with_secrets -class azurekey: +class AzureKey: def __init__(self, azure_object, key_name, load=True): self.azure_object = azure_object @@ -161,7 +166,7 @@ def generate_url(self, seconds, display_filename=None, content_type=None, inline return self.blob_client.url + '?' + token -class azuresecret: +class AzureSecret: def __init__(self, azure_object, key_vault_reference): self.azure_object = azure_object diff --git a/docassemble_base/docassemble/base/pandoc.py b/docassemble_base/docassemble/base/pandoc.py index 747c95406..daf99a332 100644 --- a/docassemble_base/docassemble/base/pandoc.py +++ b/docassemble_base/docassemble/base/pandoc.py @@ -1,3 +1,4 @@ +# pylint: disable=global-statement, consider-using-with import os import os.path import subprocess @@ -13,13 +14,18 @@ import convertapi import requests from pikepdf import Pdf -import docassemble.base.filter -import docassemble.base.functions from docassemble.base.config import daconfig +from docassemble.base.error import DAError, DAException +from docassemble.base.filter.docx import docx_filter +from docassemble.base.filter.pandoc import pdf_filter, rtf_prefilter, rtf_filter +from docassemble.base.functions import ( + package_template_filename, + standard_template_filename, +) +from docassemble.base.hooks import secure_filename, applock from docassemble.base.logger import logmessage from docassemble.base.pdfa import pdf_to_pdfa from docassemble.base.pdftk import pdf_encrypt -from docassemble.base.error import DAError, DAException style_find = re.compile(r'{\s*(\\s([1-9])[^\}]+)\\sbasedon[^\}]+heading ([0-9])', flags=re.DOTALL) @@ -32,7 +38,6 @@ PANDOC_ENGINE = '--pdf-engine=' + daconfig.get('pandoc engine', 'pdflatex') if daconfig.get('pandoc with celery', False): PANDOC_MODE = REMOTE - from docassemble.pandoc.tasks import run_pandoc # pylint: disable=import-error,no-name-in-module elif PANDOC_PATH and shutil.which(PANDOC_PATH): PANDOC_MODE = LOCAL else: @@ -42,7 +47,6 @@ LIBREOFFICE_INITIALIZED = False if daconfig.get('libreoffice with celery', False): LIBREOFFICE_MODE = REMOTE - from docassemble.libreoffice.tasks import run_libreoffice # pylint: disable=import-error,no-name-in-module elif LIBREOFFICE_PATH and shutil.which(PANDOC_PATH): LIBREOFFICE_MODE = LOCAL else: @@ -125,7 +129,7 @@ def cloudconvert_to_pdf(in_format, from_file, to_file, pdfa, password): uploaded = True if not uploaded: raise DAException("cloudconvert_to_pdf: failed to upload") - r = requests.get("https://sync.api.cloudconvert.com/v2/jobs/%s" % (resp['data']['id'],), headers=headers, timeout=60) + r = requests.get(f"https://sync.api.cloudconvert.com/v2/jobs/{resp['data']['id']}", headers=headers, timeout=60) wait_resp = r.json() if 'data' not in wait_resp: logmessage("cloudconvert_to_pdf: wait returned " + repr(r.text)) @@ -147,12 +151,12 @@ def convertapi_to_pdf(from_file, to_file): def get_pandoc_version(): - p = subprocess.Popen( + with subprocess.Popen( [PANDOC_PATH, '--version'], stdin=subprocess.PIPE, stdout=subprocess.PIPE - ) - version_content = p.communicate()[0].decode('utf-8') + ) as p: + version_content = p.communicate()[0].decode('utf-8') version_content = re.sub(r'\n.*', '', version_content) version_content = re.sub(r'^pandoc ', '', version_content) return version_content @@ -164,8 +168,8 @@ def initialize_pandoc(): global PANDOC_INITIALIZED if PANDOC_INITIALIZED: return - PANDOC_VERSION = get_pandoc_version() - if PANDOC_VERSION.startswith('1'): + pandoc_version = get_pandoc_version() + if pandoc_version.startswith('1'): PANDOC_OLD = True PANDOC_ENGINE = '--latex-engine=' + daconfig.get('pandoc engine', 'pdflatex') else: @@ -241,21 +245,21 @@ def convert_to_file(self, question): else: self.output_extension = self.output_format if self.output_format in ('rtf', 'rtf to docx') and self.template_file is None: - self.template_file = docassemble.base.functions.standard_template_filename('Legal-Template.rtf') + self.template_file = standard_template_filename('Legal-Template.rtf') if self.output_format == 'docx' and self.reference_file is None: - self.reference_file = docassemble.base.functions.standard_template_filename('Legal-Template.docx') + self.reference_file = standard_template_filename('Legal-Template.docx') if self.output_format in ('pdf', 'tex') and self.template_file is None: - self.template_file = docassemble.base.functions.standard_template_filename('Legal-Template.tex') + self.template_file = standard_template_filename('Legal-Template.tex') yaml_to_use = [] if self.output_format in ('rtf', 'rtf to docx'): # logmessage("pre input content is " + str(self.input_content)) - self.input_content = docassemble.base.filter.rtf_prefilter(self.input_content) + self.input_content = rtf_prefilter(self.input_content) # logmessage("post input content is " + str(self.input_content)) if self.output_format == 'docx': - self.input_content = docassemble.base.filter.docx_filter(self.input_content, metadata=metadata_as_dict, question=question) + self.input_content = docx_filter(self.input_content, metadata=metadata_as_dict, question=question) if self.output_format in ('pdf', 'tex'): if len(self.initial_yaml) == 0: - standard_file = docassemble.base.functions.standard_template_filename('Legal-Template.yml') + standard_file = standard_template_filename('Legal-Template.yml') if standard_file is not None: self.initial_yaml.append(standard_file) for yaml_file in self.initial_yaml: @@ -265,7 +269,7 @@ def convert_to_file(self, question): if yaml_file is not None: yaml_to_use.append(yaml_file) # logmessage("Before: " + repr(self.input_content)) - self.input_content = docassemble.base.filter.pdf_filter(self.input_content, metadata=metadata_as_dict, question=question) + self.input_content = pdf_filter(self.input_content, metadata=metadata_as_dict, question=question) # logmessage("After: " + repr(self.input_content)) if not re.search(r'[^\s]', self.input_content): self.input_content = "\\textbf{}\n" @@ -281,7 +285,7 @@ def convert_to_file(self, question): raise DAException("Could not create latex conversion directory") icc_profile_in_temp = os.path.join(tempfile.gettempdir(), 'sRGB_IEC61966-2-1_black_scaled.icc') if not os.path.isfile(icc_profile_in_temp): - shutil.copyfile(docassemble.base.functions.standard_template_filename('sRGB_IEC61966-2-1_black_scaled.icc'), icc_profile_in_temp) + shutil.copyfile(standard_template_filename('sRGB_IEC61966-2-1_black_scaled.icc'), icc_profile_in_temp) if PANDOC_MODE not in (LOCAL, REMOTE): raise DAException('LibreOffice is not available') subprocess_arguments = [PANDOC_PATH, PANDOC_ENGINE] @@ -291,12 +295,12 @@ def convert_to_file(self, question): if len(yaml_to_use) > 0: subprocess_arguments.extend(yaml_to_use) if self.template_file is not None: - subprocess_arguments.extend(['--template=%s' % self.template_file]) + subprocess_arguments.extend([f'--template={self.template_file}']) if self.reference_file is not None: if PANDOC_OLD: - subprocess_arguments.extend(['--reference-docx=%s' % self.reference_file]) + subprocess_arguments.extend([f'--reference-docx={self.reference_file}']) else: - subprocess_arguments.extend(['--reference-doc=%s' % self.reference_file]) + subprocess_arguments.extend([f'--reference-doc={self.reference_file}']) if self.output_format in ('pdf', 'tex'): subprocess_arguments.extend(['--from=markdown+raw_tex-latex_macros']) subprocess_arguments.extend(['-s', '-o', temp_outfile.name]) @@ -307,9 +311,10 @@ def convert_to_file(self, question): try: msg = subprocess.check_output(subprocess_arguments, cwd=tempfile.gettempdir(), stderr=subprocess.STDOUT).decode('utf-8', 'ignore') except subprocess.CalledProcessError as err: - raise DAException("Failed to assemble file: " + err.output.decode()) + raise DAException("Failed to assemble file: " + err.output.decode()) from err elif PANDOC_MODE == REMOTE: - result = run_pandoc.delay(subprocess_arguments[2:], tempfile.gettempdir(), mode=0).get(disable_sync_subtasks=False) + from docassemble.pandoc.tasks import run_pandoc # pylint: disable=import-error,no-name-in-module + result = run_pandoc.delay(subprocess_arguments[2:], tempfile.gettempdir(), mode=0).get(disable_sync_subtasks=False) # pylint: disable=possibly-used-before-assignment if result.ok: msg = result.content else: @@ -325,7 +330,7 @@ def convert_to_file(self, question): file_contents = the_file.read() # with open('/tmp/asdf.rtf', 'w') as deb_file: # deb_file.write(file_contents) - file_contents = docassemble.base.filter.rtf_filter(file_contents, metadata=metadata_as_dict, styles=get_rtf_styles(self.template_file), question=question) + file_contents = rtf_filter(file_contents, metadata=metadata_as_dict, styles=get_rtf_styles(self.template_file), question=question) with open(temp_outfile.name, "wb") as the_file: the_file.write(bytearray(file_contents, encoding='utf-8')) if self.output_format == 'rtf to docx': @@ -342,7 +347,7 @@ def convert_to_file(self, question): if self.output_format == 'pdf' and (self.password or self.owner_password): pdf_encrypt(self.output_filename, self.password, self.owner_password) else: - raise IOError("Failed creating file: %s" % temp_outfile.name) + raise IOError(f"Failed creating file: {temp_outfile.name}") def convert(self, question): latex_conversion_directory = os.path.join(tempfile.gettempdir(), 'conv') @@ -365,23 +370,24 @@ def convert(self, question): input_format = "markdown+smart" if self.output_format in ('pdf', 'tex'): input_format += '+raw_tex-latex_macros' - subprocess_arguments.extend(['-M', 'latextmpdir=' + os.path.join('.', 'conv'), '--from=%s' % input_format, '--to=%s' % self.output_format]) + subprocess_arguments.extend(['-M', 'latextmpdir=' + os.path.join('.', 'conv'), f'--from={input_format}', f'--to={self.output_format}']) if self.output_format == 'html': subprocess_arguments.append('--ascii') subprocess_arguments.extend(self.arguments) # logmessage("Arguments are " + str(subprocess_arguments)) if PANDOC_MODE == LOCAL: - p = subprocess.Popen( + self.output_filename = None + with subprocess.Popen( subprocess_arguments, stdin=subprocess.PIPE, stdout=subprocess.PIPE, cwd=tempfile.gettempdir() - ) - self.output_filename = None - self.output_content = p.communicate(bytearray(self.input_content, encoding='utf-8'))[0] + ) as p: + self.output_content = p.communicate(bytearray(self.input_content, encoding='utf-8'))[0] self.output_content = self.output_content.decode() elif PANDOC_MODE == REMOTE: self.output_filename = None + from docassemble.pandoc.tasks import run_pandoc # pylint: disable=import-error,no-name-in-module result = run_pandoc.delay(subprocess_arguments[2:], tempfile.gettempdir(), input_content=self.input_content, mode=1).get(disable_sync_subtasks=False) if result.ok: self.output_content = result.content @@ -394,7 +400,7 @@ def convert(self, question): def word_to_pdf(in_file, in_format, out_file, pdfa=False, password=None, owner_password=None, update_refs=False, tagged=False, filename=None, retry=True): if filename is None: filename = 'file' - filename = docassemble.base.functions.secure_filename(filename) + filename = secure_filename(filename) tempdir = tempfile.mkdtemp(prefix='SavedFile') from_file = os.path.join(tempdir, "file." + in_format) to_file = os.path.join(tempdir, "file.pdf") @@ -510,7 +516,7 @@ def word_to_pdf(in_file, in_format, out_file, pdfa=False, password=None, owner_p if use_libreoffice: start_time = time.time() if UNOCONV_AVAILABLE: - docassemble.base.functions.server.applock('obtain', 'unoconv', maxtime=6) + applock('obtain', 'unoconv', maxtime=6) logmessage("Trying unoconv with " + repr(subprocess_arguments)) try: completed_process = subprocess.run(subprocess_arguments, cwd=tempdir, timeout=120, check=False, capture_output=True) @@ -519,10 +525,10 @@ def word_to_pdf(in_file, in_format, out_file, pdfa=False, password=None, owner_p logmessage("word_to_pdf: unoconv took too long") result = 1 tries = 5 - docassemble.base.functions.server.applock('release', 'unoconv', maxtime=6) + applock('release', 'unoconv', maxtime=6) logmessage("Finished unoconv after {:.4f} seconds.".format(time.time() - start_time)) elif UNOCONVERT_AVAILABLE: - docassemble.base.functions.server.applock('obtain', 'unoconvert', maxtime=6) + applock('obtain', 'unoconvert', maxtime=6) logmessage("Trying unoconvert with " + repr(subprocess_arguments)) try: completed_process = subprocess.run(subprocess_arguments, cwd=tempdir, timeout=120, check=False, capture_output=True) @@ -531,12 +537,12 @@ def word_to_pdf(in_file, in_format, out_file, pdfa=False, password=None, owner_p logmessage("word_to_pdf: unoconvert took too long") result = 1 tries = 5 - docassemble.base.functions.server.applock('release', 'unoconvert', maxtime=6) + applock('release', 'unoconvert', maxtime=6) logmessage("Finished unoconvert after {:.4f} seconds.".format(time.time() - start_time)) elif LIBREOFFICE_MODE == LOCAL: initialize_libreoffice() logmessage("Trying libreoffice with " + repr(subprocess_arguments)) - docassemble.base.functions.server.applock('obtain', 'libreoffice') + applock('obtain', 'libreoffice') logmessage("Obtained libreoffice lock after {:.4f} seconds.".format(time.time() - start_time)) try: completed_process = subprocess.run(subprocess_arguments, cwd=tempdir, timeout=120, check=False, capture_output=True) @@ -546,9 +552,10 @@ def word_to_pdf(in_file, in_format, out_file, pdfa=False, password=None, owner_p result = 1 tries = 5 logmessage("Finished libreoffice after {:.4f} seconds.".format(time.time() - start_time)) - docassemble.base.functions.server.applock('release', 'libreoffice') + applock('release', 'libreoffice') elif LIBREOFFICE_MODE == REMOTE: - result = run_libreoffice.delay(subprocess_arguments[1:], tempfile.gettempdir()).get(disable_sync_subtasks=False) + from docassemble.libreoffice.tasks import run_libreoffice # pylint: disable=import-error,no-name-in-module + result = run_libreoffice.delay(subprocess_arguments[1:], tempfile.gettempdir()).get(disable_sync_subtasks=False) # pylint: disable=possibly-used-before-assignment if result == 1234: result = 1 tries = 5 @@ -638,17 +645,18 @@ def rtf_to_docx(in_file, out_file): if result != 0: logmessage("rtf_to_docx: call to unoconvert returned non-zero response") elif LIBREOFFICE_MODE == LOCAL: - docassemble.base.functions.server.applock('obtain', 'libreoffice') + applock('obtain', 'libreoffice') try: result = subprocess.run(subprocess_arguments, cwd=tempdir, timeout=120, check=False).returncode except subprocess.TimeoutExpired: logmessage("rtf_to_docx: call to LibreOffice took too long") result = 1 tries = 5 - docassemble.base.functions.server.applock('release', 'libreoffice') + applock('release', 'libreoffice') if result != 0: logmessage("rtf_to_docx: call to LibreOffice returned non-zero response") else: + from docassemble.libreoffice.tasks import run_libreoffice # pylint: disable=import-error,no-name-in-module result = run_libreoffice.delay(subprocess_arguments[1:], tempfile.gettempdir()).get(disable_sync_subtasks=False) if result == 1234: result = 1 @@ -712,17 +720,18 @@ def convert_file(in_file, out_file, input_extension, output_extension): if result != 0: logmessage("convert_file: call to unoconvert returned non-zero response") elif LIBREOFFICE_MODE == LOCAL: - docassemble.base.functions.server.applock('obtain', 'libreoffice') + applock('obtain', 'libreoffice') try: result = subprocess.run(subprocess_arguments, cwd=tempdir1, timeout=120, check=False).returncode except subprocess.TimeoutExpired: logmessage("convert_file: libreoffice took too long") result = 1 tries = 5 - docassemble.base.functions.server.applock('release', 'libreoffice') + applock('release', 'libreoffice') if result != 0: logmessage("convert_file: call to LibreOffice returned non-zero response") else: + from docassemble.libreoffice.tasks import run_libreoffice # pylint: disable=import-error,no-name-in-module result = run_libreoffice.delay(subprocess_arguments[1:], tempfile.gettempdir()).get(disable_sync_subtasks=False) if result == 1234: result = 1 @@ -796,17 +805,18 @@ def word_to_markdown(in_file, in_format): if result != 0: logmessage("word_to_markdown: call to unoconvert returned non-zero response") elif LIBREOFFICE_MODE == LOCAL: - docassemble.base.functions.server.applock('obtain', 'libreoffice') + applock('obtain', 'libreoffice') try: result = subprocess.run(subprocess_arguments, cwd=tempdir, timeout=120, check=False).returncode except subprocess.TimeoutExpired: logmessage("word_to_markdown: libreoffice took too long") result = 1 tries = 5 - docassemble.base.functions.server.applock('release', 'libreoffice') + applock('release', 'libreoffice') if result != 0: logmessage("word_to_markdown: call to LibreOffice returned non-zero response") elif LIBREOFFICE_MODE == REMOTE: + from docassemble.libreoffice.tasks import run_libreoffice # pylint: disable=import-error,no-name-in-module result = run_libreoffice.delay(subprocess_arguments[1:], tempfile.gettempdir()).get(disable_sync_subtasks=False) if result == 1234: result = 1 @@ -839,13 +849,14 @@ def word_to_markdown(in_file, in_format): else: if in_format_to_use == 'markdown': in_format_to_use = "markdown+smart" - subprocess_arguments.extend(['--from=%s' % str(in_format_to_use), '--to=markdown_phpextra', str(in_file_to_use), '-o', str(temp_file.name)]) + subprocess_arguments.extend([f'--from={in_format_to_use}', '--to=markdown_phpextra', str(in_file_to_use), '-o', str(temp_file.name)]) if PANDOC_MODE == LOCAL: try: result = subprocess.run(subprocess_arguments, timeout=60, check=False).returncode except subprocess.TimeoutExpired: result = 1 elif PANDOC_MODE == REMOTE: + from docassemble.pandoc.tasks import run_pandoc # pylint: disable=import-error,no-name-in-module result = run_pandoc.delay(subprocess_arguments[2:], tempfile.gettempdir(), mode=2).get(disable_sync_subtasks=False) else: raise DAException("Pandoc not installed.") @@ -890,14 +901,15 @@ def update_references(filename): tries = 0 while tries < 5: if LIBREOFFICE_MODE == LOCAL: - docassemble.base.functions.server.applock('obtain', 'libreoffice') + applock('obtain', 'libreoffice') try: result = subprocess.run(subprocess_arguments, cwd=tempfile.gettempdir(), timeout=120, check=False).returncode except subprocess.TimeoutExpired: result = 1 tries = 5 - docassemble.base.functions.server.applock('release', 'libreoffice') + applock('release', 'libreoffice') else: + from docassemble.libreoffice.tasks import run_libreoffice # pylint: disable=import-error,no-name-in-module result = run_libreoffice.delay(subprocess_arguments[1:], tempfile.gettempdir()).get(disable_sync_subtasks=False) if result == 1234: result = 1 @@ -922,11 +934,11 @@ def initialize_libreoffice(): if not os.path.isfile(LIBREOFFICE_MACRO_PATH): logmessage("No LibreOffice macro path exists") temp_file = tempfile.NamedTemporaryFile(prefix="datemp", mode="wb", suffix=".pdf") - word_file = docassemble.base.functions.package_template_filename('docassemble.demo:data/templates/template_test.docx') + word_file = package_template_filename('docassemble.demo:data/templates/template_test.docx') word_to_pdf(word_file, 'docx', temp_file.name, pdfa=False, password=None, owner_password=None, retry=False) del temp_file del word_file - orig_path = docassemble.base.functions.package_template_filename('docassemble.base:data/macros/Module1.xba') + orig_path = package_template_filename('docassemble.base:data/macros/Module1.xba') try: assert os.path.isdir(os.path.dirname(LIBREOFFICE_MACRO_PATH)) # logmessage("Copying LibreOffice macro from " + orig_path) diff --git a/docassemble_base/docassemble/base/parse.py b/docassemble_base/docassemble/base/parse.py index 18ed5ca84..d2420a7c0 100644 --- a/docassemble_base/docassemble/base/parse.py +++ b/docassemble_base/docassemble/base/parse.py @@ -6,7 +6,6 @@ import os import os.path import sys -import datetime import time import operator import pprint @@ -15,53 +14,116 @@ import array import tempfile import json -import platform import textwrap from urllib.request import urlretrieve from io import StringIO -from collections import abc, OrderedDict, namedtuple +from collections import abc, OrderedDict from types import CodeType, FunctionType import xml.etree.ElementTree as ET from html.parser import HTMLParser -from itertools import groupby, chain import ruamel.yaml -from jinja2 import ChainableUndefined -from jinja2.runtime import StrictUndefined, UndefinedError +from jinja2.runtime import UndefinedError from jinja2.exceptions import TemplateError -from jinja2.environment import Environment -from jinja2 import FileSystemLoader, select_autoescape, TemplateNotFound from jinja2 import meta as jinja2meta -from jinja2.lexer import Token -from jinja2.utils import internalcode, missing, object_type_repr -from jinja2.ext import Extension from docxtpl import DocxTemplate import dateutil.parser try: import zoneinfo except ImportError: - from backports import zoneinfo + from backports import zoneinfo # type: ignore[no-redef] from bs4 import BeautifulSoup from docassemble_textstat.textstat import textstat import qrcode import qrcode.image.svg -from docassemble.base import __version__ as da_version -import docassemble.base.filter -import docassemble.base.pdftk -import docassemble.base.file_docx -from docassemble.base.error import DAError, DANotFoundError, MandatoryQuestion, DAErrorNoEndpoint, DAErrorMissingVariable, ForcedNameError, QuestionError, ResponseError, BackgroundResponseError, BackgroundResponseActionError, CommandError, CodeExecute, DAValidationError, ForcedReRun, LazyNameError, DAAttributeError, DAIndexError, DAException, DANameError, DASourceError -import docassemble.base.functions -import docassemble.base.util -from docassemble.base.functions import pickleable_objects, word, get_language, RawValue, get_config, safeyaml, altyaml, prettyyaml -from docassemble.base.logger import logmessage -from docassemble.base.pandoc import MyPandoc -from docassemble.base.mako.template import Template as MakoTemplate -from docassemble.base.mako.exceptions import SyntaxException, CompileException -from docassemble.base.astparser import myvisitnode +from .astparser import DetectIllegal +from .astparser import MyVisitNode +from .dates import format_date +from .error import ( + BackgroundResponseActionError, + BackgroundResponseError, + CodeExecute, + CommandError, + DAAttributeError, + DAError, + DAErrorMissingVariable, + DAErrorNoEndpoint, + DAException, + DAIndexError, + DANameError, + DANotFoundError, + DASourceError, + DAValidationError, + ForcedNameError, + ForcedReRun, + LazyNameError, + MandatoryQuestion, + QuestionError, + ResponseError, +) +from .interview_source import interview_source_from_string, InterviewSourceString +from .file_docx import transform_for_docx, concatenate_files, fix_docx +from .filter.html import ( + emoji_html, + emoji_insert, + get_audio_urls, + get_video_urls, + markdown_to_html, +) +from .filter.utils import convert_svg_to_png +from .functions import ( + DANav, + RawValue, + altyaml, + custom_types, + get_action_stack, + get_config, + get_language, + intrinsic_name_of, + package_data_filename, + package_template_filename, + pickleable_objects, + pop_current_variable, + pop_event_stack, + prettyyaml, + reconsider, + reset_context, + reset_gathering_mode, + safeyaml, + serializable_dict, + set_context, + set_current_variable, + single_paragraph, + space_to_underscore, + static_filename_path, + undefine, + url_action, + word, + wrap_up, +) +from .helpers import extract_missing_name, fix_quotes +from .hooks import ( + file_finder, + get_default_language, + get_default_timezone, + get_main_page_parts, + save_numbered_file, + secure_filename_unicode_ok, + to_text, + url_finder, +) +from .jinja import custom_jinja_env +from .language.control import set_language +from .logger import logmessage +from .mako.exceptions import SyntaxException, CompileException +from .mako.template import Template as MakoTemplate +from .pandoc import MyPandoc, word_to_pdf, update_references +from .pdftk import fill_template +from .thread_context import this_thread +from .util import objects_from_structure, objects_from_data equals_byte = bytes('=', 'utf-8') RangeType = type(range(1, 2)) NoneType = type(None) -da_arch = platform.machine() standard_types = set(['integer', 'number', 'currency', 'float', 'file', 'files', 'range', 'multiselect', 'checkboxes', 'object_multiselect', 'object_checkboxes', 'user', 'camera', 'environment', 'date', 'datetime', 'time', 'email', 'microphone', 'ml', 'mlarea', 'noyes', 'noyesmaybe', 'noyesradio', 'noyeswide', 'yesno', 'yesnomaybe', 'yesnoradio', 'yesnowide', 'text', 'password', 'object']) DEBUG = True @@ -73,7 +135,6 @@ match_mako = re.compile(r'<%|\${|% if|% for|% while|\#\#') emoji_match = re.compile(r':([^ ]+):') valid_variable_match = re.compile(r'^[^\d][A-Za-z0-9\_]*$') -nameerror_match = re.compile(r'\'(.*)\' (is not defined|referenced before assignment|is undefined|where it is not)') document_match = re.compile(r'^--- *$', flags=re.MULTILINE) remove_trailing_dots = re.compile(r'[\n\r]+\.\.\.$') fix_tabs = re.compile(r'\t') @@ -103,20 +164,20 @@ def textify(data, the_user_dict): # def set_absolute_filename(func): # # logmessage("Running set_absolute_filename in parse") -# docassemble.base.functions.set_absolute_filename(func) +# set_absolute_filename(func) # def set_url_finder(func): -# docassemble.base.filter.set_url_finder(func) -# docassemble.base.functions.set_url_finder(func) +# set_url_finder(func) +# set_url_finder(func) # def set_url_for(func): -# docassemble.base.filter.set_url_for(func) +# set_url_for(func) # def set_file_finder(func): -# docassemble.base.filter.set_file_finder(func) +# set_file_finder(func) # def set_da_send_mail(func): -# docassemble.base.filter.set_da_send_mail(func) +# set_da_send_mail(func) # def blank_save_numbered_file(*args, **kwargs): # return (None, None, None) @@ -128,7 +189,7 @@ def textify(data, the_user_dict): # # logmessage("set the save_numbered_file function to " + str(func)) # save_numbered_file = func -INITIAL_DICT = {'_internal': {'session_local': {}, 'device_local': {}, 'user_local': {}, 'dirty': {}, 'progress': 0, 'tracker': 0, 'docvar': {}, 'doc_cache': {}, 'steps': 1, 'steps_offset': 0, 'secret': None, 'informed': {}, 'livehelp': {'availability': 'unavailable', 'mode': 'help', 'roles': [], 'partner_roles': []}, 'answered': set(), 'answers': {}, 'objselections': {}, 'starttime': None, 'modtime': None, 'accesstime': {}, 'tasks': {}, 'gather': [], 'event_stack': {}, 'misc': {}}, 'url_args': {}, 'nav': docassemble.base.functions.DANav()} +INITIAL_DICT = {'_internal': {'session_local': {}, 'device_local': {}, 'user_local': {}, 'dirty': {}, 'progress': 0, 'tracker': 0, 'docvar': {}, 'doc_cache': {}, 'steps': 1, 'steps_offset': 0, 'secret': None, 'informed': {}, 'livehelp': {'availability': 'unavailable', 'mode': 'help', 'roles': [], 'partner_roles': []}, 'answered': set(), 'answers': {}, 'objselections': {}, 'starttime': None, 'modtime': None, 'accesstime': {}, 'tasks': {}, 'gather': [], 'event_stack': {}, 'misc': {}}, 'url_args': {}, 'nav': DANav()} def set_initial_dict(the_dict): @@ -140,27 +201,6 @@ def get_initial_dict(): return copy.deepcopy(INITIAL_DICT) -class DAFileSystemLoader(FileSystemLoader): - - def get_source(self, environment, template): - if ':' not in template: - return super().get_source(environment, template) - template_path = None - for the_filename in question_path_options(template): - if the_filename is not None: - template_path = the_filename - break - if template_path is None or not os.path.isfile(template_path): - raise TemplateNotFound(template) - fspath = os.fspath(os.path.dirname(template_path)) - if fspath not in self.searchpath: - self.searchpath.append(fspath) - mtime = os.path.getmtime(template_path) - with open(template_path, 'r', encoding='utf-8') as fp: - source = fp.read() - return source, template_path, lambda: mtime == os.path.getmtime(template_path) - - class PackageImage: def __init__(self, **kwargs): @@ -170,261 +210,13 @@ def __init__(self, **kwargs): self.package = kwargs.get('package', 'docassemble.base') def get_filename(self): - return docassemble.base.functions.static_filename_path(str(self.package) + ':' + str(self.filename)) + return static_filename_path(str(self.package) + ':' + str(self.filename)) def get_reference(self): # logmessage("get_reference is considering " + str(self.package) + ':' + str(self.filename)) return str(self.package) + ':' + str(self.filename) -class InterviewSource: - - def __init__(self, **kwargs): - if not hasattr(self, 'package'): - self.package = kwargs.get('package', None) - self.language = kwargs.get('language', '*') - self.dialect = kwargs.get('dialect', None) - self.testing = kwargs.get('testing', False) - self.translating = kwargs.get('translating', False) - - def __le__(self, other): - return str(self) <= (str(other) if isinstance(other, InterviewSource) else other) - - def __ge__(self, other): - return str(self) >= (str(other) if isinstance(other, InterviewSource) else other) - - def __gt__(self, other): - return str(self) > (str(other) if isinstance(other, InterviewSource) else other) - - def __lt__(self, other): - return str(self) < (str(other) if isinstance(other, InterviewSource) else other) - - def __eq__(self, other): - return self is other - - def __ne__(self, other): - return self is not other - - def __str__(self): - if hasattr(self, 'path'): - return str(self.path) - return 'interviewsource' - - def __hash__(self): - if hasattr(self, 'path'): - return hash((self.path,)) - return hash(('interviewsource',)) - - def set_path(self, path): - self.path = path - - def get_name(self): - if ':' in self.path: - return self.path - return self.get_package() + ':data/questions/' + self.path - - def get_index(self): - the_index = docassemble.base.functions.server.server_redis.get('da:interviewsource:' + self.path) - if the_index is None: - # logmessage("Updating index from get_index for " + self.path) - the_index = docassemble.base.functions.server.server_redis.incr('da:interviewsource:' + self.path) - return the_index - - def update_index(self): - # logmessage("Updating index for " + self.path) - docassemble.base.functions.server.server_redis.incr('da:interviewsource:' + self.path) - - def set_filepath(self, filepath): - self.filepath = filepath - - def set_directory(self, directory): - self.directory = directory - - def set_content(self, content): - self.content = content - - def set_language(self, language): - self.language = language - - def set_dialect(self, dialect): - self.dialect = dialect - - def set_testing(self, testing): - self.testing = testing - - def set_package(self, package): - self.package = package - - def update(self, **kwargs): # pylint: disable=unused-argument - return True - - def get_modtime(self): - return self._modtime - - def get_language(self): - return self.language - - def get_dialect(self): - return self.dialect - - def get_package(self): - return self.package - - def get_testing(self): - return self.testing - - def get_interview(self): - return Interview(source=self) - - def append(self, path): # pylint: disable=unused-argument - return None - - -class InterviewSourceString(InterviewSource): - - def __init__(self, **kwargs): - self.set_path(kwargs.get('path', None)) - self.set_directory(kwargs.get('directory', None)) - self.set_content(kwargs.get('content', None)) - self._modtime = datetime.datetime.now(tz=datetime.timezone.utc) - super().__init__(**kwargs) - - -class InterviewSourceFile(InterviewSource): - - def __init__(self, **kwargs): - self.playground = None - if 'filepath' in kwargs: - if kwargs['filepath'].__class__.__name__.endswith('SavedFile'): - self.playground = kwargs['filepath'] - if self.playground.subdir and self.playground.subdir != 'default': - self.playground_file = os.path.join(self.playground.subdir, self.playground.filename) - else: - self.playground_file = self.playground.filename - # logmessage("The path is " + repr(self.playground.path)) - if os.path.isfile(self.playground.path) and os.access(self.playground.path, os.R_OK): - self.set_filepath(self.playground.path) - else: - logmessage("Details of playground path reference:") - logmessage("Keyword arguments were " + repr(kwargs)) - for attribute in ['file_number', 'fixed', 'section', 'filename', 'extension', 'directory', 'path', 'modtimes', 'keydict', 'subdir']: - if hasattr(self.playground, attribute): - logmessage(attribute + " is " + repr(getattr(self.playground, attribute))) - else: - logmessage(attribute + " did not exist") - if os.path.exists(self.playground.path): - if os.path.isfile(self.playground.path): - if os.access(self.playground.path, os.R_OK): - logmessage("path is a file and is readable") - else: - logmessage("path is a file but is not readable") - else: - logmessage("path was not a file") - else: - logmessage("path did not exist") - raise DANotFoundError("Reference to invalid playground path.") - else: - self.set_filepath(kwargs['filepath']) - else: - self.filepath = None - if 'path' in kwargs: - self.set_path(kwargs['path']) - super().__init__(**kwargs) - - def set_path(self, path): - self.path = path - parts = path.split(":") - if len(parts) == 2: - self.package = parts[0] - self.basename = parts[1] - else: - self.package = None - # if self.package is None: - # m = re.search(r'^/(playground\.[0-9]+)/', path) - # if m: - # self.package = m.group(1) - if self.filepath is None: - self.set_filepath(interview_source_from_string(self.path)) - if self.package is None and re.search(r'docassemble.base.data.', self.filepath): - self.package = 'docassemble.base' - - def set_filepath(self, filepath): - # logmessage("Called set_filepath with " + str(filepath)) - self.filepath = filepath - if self.filepath is None: - self.directory = None - else: - self.set_directory(os.path.dirname(self.filepath)) - - def reset_modtime(self): - try: - with open(self.filepath, 'a', encoding='utf-8'): - os.utime(self.filepath, None) - except: - logmessage("InterviewSourceFile: could not reset modification time on interview") - - def update(self, **kwargs): - try: - with open(self.filepath, 'r', encoding='utf-8') as the_file: - orig_text = the_file.read() - except: - return False - if not orig_text.startswith('# use jinja'): - self.set_content(orig_text) - return True - env = Environment( - loader=DAFileSystemLoader(self.directory), - autoescape=select_autoescape() - ) - if kwargs.get('raise_jinja_errors', True): - template = env.get_template(os.path.basename(self.filepath)) - else: - try: - template = env.get_template(os.path.basename(self.filepath)) - except TemplateError: - self.set_content(orig_text) - return True - data = copy.deepcopy(get_config('jinja data')) - data['__config__'] = copy.deepcopy(docassemble.base.functions.server.daconfig) - data['__version__'] = da_version - data['__architecture__'] = da_arch - data['__filename__'] = self.path - data['__current_package__'] = self.package - data['__parent_filename__'] = kwargs.get('parent_source', self).path - data['__parent_package__'] = kwargs.get('parent_source', self).package - data['__interview_filename__'] = kwargs.get('interview_source', self).path - data['__interview_package__'] = kwargs.get('interview_source', self).package - data['__hostname__'] = get_config('external hostname', None) or 'localhost' - data['__debug__'] = bool(get_config('debug', True)) - try: - self.set_content(template.render(data)) - except BaseException as err: - self.set_content("__error__: " + repr("Jinja2 rendering error: " + err.__class__.__name__ + ": " + str(err))) - return True - - def get_modtime(self): - # logmessage("get_modtime called in parse where path is " + str(self.path)) - if self.playground is not None: - return self.playground.get_modtime(filename=self.playground_file) - self._modtime = os.path.getmtime(self.filepath) - return self._modtime - - def append(self, path): - new_file = os.path.join(self.directory, path) - if os.path.isfile(new_file) and os.access(new_file, os.R_OK): - new_source = InterviewSourceFile() - new_source.path = path - new_source.directory = self.directory - new_source.basename = path - new_source.filepath = new_file - new_source.playground = self.playground - if hasattr(self, 'package'): - new_source.package = self.package - if new_source.update(): - return new_source - return None - - def dummy_embed_input(status, variable): # pylint: disable=unused-argument return variable @@ -781,13 +573,13 @@ def initialize_screen_reader(self): def populate(self, question_result): self.question = question_result['question'] - self.questionText = question_result['question_text'] - self.subquestionText = question_result['subquestion_text'] - self.continueLabel = question_result['continue_label'] + self.question_text = question_result['question_text'] + self.subquestion_text = question_result['subquestion_text'] + self.continue_label = question_result['continue_label'] self.decorations = question_result['decorations'] self.audiovideo = question_result['audiovideo'] - self.helpText = question_result['help_text'] or [] - self.interviewHelpText = question_result['interview_help_text'] or [] + self.help_text = question_result['help_text'] or [] + self.interview_help_text = question_result['interview_help_text'] or [] self.attachments = question_result['attachments'] or [] self.selectcompute = question_result['selectcompute'] self.defaults = question_result['defaults'] @@ -860,8 +652,8 @@ def get_history(self): def convert_help(self, help_text, encode, debug): the_help = {} if 'audiovideo' in help_text and help_text['audiovideo'] is not None: - audio_result = docassemble.base.filter.get_audio_urls(help_text['audiovideo']) - video_result = docassemble.base.filter.get_video_urls(help_text['audiovideo']) + audio_result = get_audio_urls(help_text['audiovideo']) + video_result = get_video_urls(help_text['audiovideo']) if len(audio_result) > 0: the_help['audio'] = [{'url': x[0], 'mime_type': x[1]} for x in audio_result] if len(video_result) > 0: @@ -873,12 +665,12 @@ def convert_help(self, help_text, encode, debug): the_help['help'] = '' the_help['help'] += '

' + the_help['heading'] + '

' if 'content' in help_text and help_text['content'] is not None: - the_help['content'] = docassemble.base.filter.markdown_to_html(help_text['content'].rstrip(), status=self, verbatim=(not encode)) + the_help['content'] = markdown_to_html(help_text['content'].rstrip(), status=self, verbatim=not encode) if debug: if 'help' not in the_help: the_help['help'] = '' the_help['help'] += '

' + the_help['content'] + '

' - # elif len(self.helpText) > 1: + # elif len(self.help_text) > 1: # the_help['heading'] = word('Help with this question') return the_help @@ -895,7 +687,7 @@ def as_data(self, the_user_dict, encode=True): result['validation_messages'] = {} if 'reload_after' in self.extras: result['reload'] = 1000 * int(self.extras['reload_after']) - lang = docassemble.base.functions.get_language() + lang = get_language() if len(self.question.terms) > 0 or len(self.question.interview.terms) > 0: result['terms'] = {} if 'terms' in self.extras: @@ -924,29 +716,28 @@ def as_data(self, the_user_dict, encode=True): result['additional_buttons'] = [] for item in self.extras['action_buttons']: new_item = copy.deepcopy(item) - new_item['label'] = docassemble.base.filter.markdown_to_html(item['label'], trim=True, do_terms=False, status=self, verbatim=(not encode)) + new_item['label'] = markdown_to_html(item['label'], trim=True, do_terms=False, status=self, verbatim=not encode) if debug: output['question'] += '

' + new_item['label'] + '

' - for param in ('questionText',): - if hasattr(self, param) and getattr(self, param) is not None: - result[param] = docassemble.base.filter.markdown_to_html(getattr(self, param).rstrip(), trim=True, status=self, verbatim=(not encode)) - if debug: - output['question'] += '

' + result[param] + '

' + if hasattr(self, 'question_text') and getattr(self, 'question_text') is not None: + result['questionText'] = markdown_to_html(getattr(self, 'question_text').rstrip(), trim=True, status=self, verbatim=not encode) + if debug: + output['question'] += '

' + result['questionText'] + '

' if debug: if hasattr(self, 'breadcrumb') and self.breadcrumb is not None: output['breadcrumb label'] = self.breadcrumb - output['breadcrumbs'] = docassemble.base.functions.get_action_stack() - if hasattr(self, 'subquestionText') and self.subquestionText is not None: + output['breadcrumbs'] = get_action_stack() + if hasattr(self, 'subquestion_text') and self.subquestion_text is not None: if self.question.question_type == "fields": embedder = dummy_embed_input else: embedder = None - result['subquestionText'] = docassemble.base.filter.markdown_to_html(self.subquestionText.rstrip(), status=self, verbatim=(not encode), embedder=embedder) + result['subquestionText'] = markdown_to_html(self.subquestion_text.rstrip(), status=self, verbatim=not encode, embedder=embedder) if debug: output['question'] += result['subquestionText'] - for param in ('continueLabel', 'helpLabel'): - if hasattr(self, param) and getattr(self, param) is not None: - result[param] = docassemble.base.filter.markdown_to_html(getattr(self, param).rstrip(), trim=True, do_terms=False, status=self, verbatim=(not encode)) + for attr, param in (('continue_label', 'continueLabel'), ('help_label', 'helpLabel')): + if hasattr(self, attr) and getattr(self, attr) is not None: + result[param] = markdown_to_html(getattr(self, attr).rstrip(), trim=True, do_terms=False, status=self, verbatim=not encode) if debug: output['question'] += '

' + result[param] + '

' if 'menu_items' in self.extras and isinstance(self.extras['menu_items'], list): @@ -956,10 +747,10 @@ def as_data(self, the_user_dict, encode=True): result[param] = self.extras[param].rstrip() for param in ('back_button_label',): if param in self.extras and isinstance(self.extras[param], str): - result[param] = docassemble.base.filter.markdown_to_html(self.extras[param].rstrip(), trim=True, do_terms=False, status=self, verbatim=(not encode)) + result[param] = markdown_to_html(self.extras[param].rstrip(), trim=True, do_terms=False, status=self, verbatim=not encode) for param in ('rightText', 'underText'): if param in self.extras and isinstance(self.extras[param], str): - result[param] = docassemble.base.filter.markdown_to_html(self.extras[param].rstrip(), status=self, verbatim=(not encode)) + result[param] = markdown_to_html(self.extras[param].rstrip(), status=self, verbatim=not encode) if debug: output['question'] += result[param] if 'continueLabel' not in result: @@ -1007,20 +798,20 @@ def as_data(self, the_user_dict, encode=True): if hasattr(self.question, 'id'): result['id'] = self.question.id if hasattr(self, 'audiovideo') and self.audiovideo is not None: - audio_result = docassemble.base.filter.get_audio_urls(self.audiovideo) - video_result = docassemble.base.filter.get_video_urls(self.audiovideo) + audio_result = get_audio_urls(self.audiovideo) + video_result = get_video_urls(self.audiovideo) if len(audio_result) > 0: result['audio'] = [{'url': re.sub(r'.*"(http[^"]+)".*', r'\1', x)} if isinstance(x, str) else {'url': x[0], 'mime_type': x[1]} for x in audio_result] if len(video_result) > 0: result['video'] = [{'url': re.sub(r'.*"(http[^"]+)".*', r'\1', x)} if isinstance(x, str) else {'url': x[0], 'mime_type': x[1]} for x in video_result] - if hasattr(self, 'helpText') and len(self.helpText) > 0: + if hasattr(self, 'help_text') and len(self.help_text) > 0: result['helpText'] = [] result['helpBackLabel'] = word("Back to question") - for help_text in self.helpText: + for help_text in self.help_text: result['helpText'].append(self.convert_help(help_text, encode, debug)) result['help'] = {} - if self.helpText[0]['label']: - result['help']['label'] = docassemble.base.filter.markdown_to_html(self.helpText[0]['label'], trim=True, do_terms=False, status=self, verbatim=(not encode)) + if self.help_text[0]['label']: + result['help']['label'] = markdown_to_html(self.help_text[0]['label'], trim=True, do_terms=False, status=self, verbatim=not encode) else: result['help']['label'] = self.question.help() result['help']['title'] = word("Help is available for this question") @@ -1033,9 +824,9 @@ def as_data(self, the_user_dict, encode=True): if 'help' in item: output['help'] += '
' + item['help'] + '
' output['help'] += '' - if hasattr(self, 'interviewHelpText') and len(self.interviewHelpText) > 0: + if hasattr(self, 'interview_help_text') and len(self.interview_help_text) > 0: result['interviewHelpText'] = [] - for help_text in self.interviewHelpText: + for help_text in self.interview_help_text: result['interviewHelpText'].append(self.convert_help(help_text, encode, debug)) if debug: for item in result['interviewHelpText']: @@ -1043,12 +834,12 @@ def as_data(self, the_user_dict, encode=True): output['help'] += '
' + item['help'] + '
' if 'help' not in result: result['help'] = {} - if self.interviewHelpText[0]['label']: - result['help']['interviewLabel'] = docassemble.base.filter.markdown_to_html(self.interviewHelpText[0]['label'], trim=True, do_terms=False, status=self, verbatim=(not encode)) + if self.interview_help_text[0]['label']: + result['help']['interviewLabel'] = markdown_to_html(self.interview_help_text[0]['label'], trim=True, do_terms=False, status=self, verbatim=not encode) else: result['help']['interviewLabel'] = self.question.help() result['help']['interviewTitle'] = word("Help is available") - if not (hasattr(self, 'helpText') and len(self.helpText) > 0): + if not (hasattr(self, 'help_text') and len(self.help_text) > 0): result['help']['specific'] = False if 'questionText' not in result and self.question.question_type == "signature": result['questionText'] = '

' + word('Sign Your Name') + '

' @@ -1080,9 +871,9 @@ def as_data(self, the_user_dict, encode=True): result['decoration'] = {} the_image = self.question.interview.images.get(decoration['image'], None) if the_image is not None: - the_url = docassemble.base.functions.server.url_finder(str(the_image.package) + ':' + str(the_image.filename)) + the_url = url_finder(str(the_image.package) + ':' + str(the_image.filename)) width = str(width_value) + str(width_units) - filename = docassemble.base.functions.server.file_finder(str(the_image.package) + ':' + str(the_image.filename)) + filename = file_finder(str(the_image.package) + ':' + str(the_image.filename)) if 'extension' in filename and filename['extension'] == 'svg' and 'width' in filename: if filename['width'] and filename['height']: height = str(width_value * (filename['height']/filename['width'])) + str(width_units) @@ -1110,19 +901,19 @@ def as_data(self, the_user_dict, encode=True): the_attachment['variable_name'] = attachment['orig_variable_name'] if 'name' in attachment: if attachment['name']: - the_attachment['name'] = docassemble.base.filter.markdown_to_html(attachment['name'], trim=True, status=self, verbatim=(not encode)) + the_attachment['name'] = markdown_to_html(attachment['name'], trim=True, status=self, verbatim=not encode) if debug: output['question'] += '

' + the_attachment['name'] + '

' if 'description' in attachment: if attachment['description']: - the_attachment['description'] = docassemble.base.filter.markdown_to_html(attachment['description'], status=self, verbatim=(not encode)) + the_attachment['description'] = markdown_to_html(attachment['description'], status=self, verbatim=not encode) if debug: output['question'] += '

' + the_attachment['description'] + '

' for key in ('valid_formats', 'filename', 'content', 'markdown', 'raw'): if key in attachment and attachment[key]: the_attachment[key] = attachment[key] for the_format in attachment['file']: - the_attachment['url'][the_format] = docassemble.base.functions.server.url_finder(attachment['file'][the_format], filename=attachment['filename'] + '.' + extension_of_doc_format.get(the_format, the_format)) + the_attachment['url'][the_format] = url_finder(attachment['file'][the_format], filename=attachment['filename'] + '.' + extension_of_doc_format.get(the_format, the_format)) the_attachment['number'][the_format] = attachment['file'][the_format] the_attachment['filename_with_extension'][the_format] = attachment['filename'] + '.' + extension_of_doc_format.get(the_format, the_format) result['attachments'].append(the_attachment) @@ -1210,26 +1001,26 @@ def as_data(self, the_user_dict, encode=True): if field.datatype == 'date': the_field['validation_messages']['date'] = field.validation_message('date', self, word("You need to enter a valid date.")) if hasattr(field, 'extras') and 'min' in field.extras and 'min' in self.extras and 'max' in field.extras and 'max' in self.extras and field.number in self.extras['min'] and field.number in self.extras['max']: - the_field['validation_messages']['minmax'] = field.validation_message('date minmax', self, word("You need to enter a date between %s and %s."), parameters=(docassemble.base.util.format_date(self.extras['min'][field.number], format='medium'), docassemble.base.util.format_date(self.extras['max'][field.number], format='medium'))) + the_field['validation_messages']['minmax'] = field.validation_message('date minmax', self, word("You need to enter a date between %s and %s."), parameters=(format_date(self.extras['min'][field.number], format='medium'), format_date(self.extras['max'][field.number], format='medium'))) else: was_defined = {} for key in ['min', 'max']: if hasattr(field, 'extras') and key in field.extras and key in self.extras and field.number in self.extras[key]: was_defined[key] = True if key == 'min': - the_field['validation_messages']['min'] = field.validation_message('date min', self, word("You need to enter a date on or after %s."), parameters=tuple([docassemble.base.util.format_date(self.extras[key][field.number], format='medium')])) + the_field['validation_messages']['min'] = field.validation_message('date min', self, word("You need to enter a date on or after %s."), parameters=tuple([format_date(self.extras[key][field.number], format='medium')])) elif key == 'max': - the_field['validation_messages']['max'] = field.validation_message('date max', self, word("You need to enter a date on or before %s."), parameters=tuple([docassemble.base.util.format_date(self.extras[key][field.number], format='medium')])) + the_field['validation_messages']['max'] = field.validation_message('date max', self, word("You need to enter a date on or before %s."), parameters=tuple([format_date(self.extras[key][field.number], format='medium')])) if len(was_defined) == 0 and 'default date min' in self.question.interview.options and 'default date max' in self.question.interview.options: - the_field['min'] = docassemble.base.util.format_date(self.question.interview.options['default date min'], format='yyyy-MM-dd') - the_field['max'] = docassemble.base.util.format_date(self.question.interview.options['default date max'], format='yyyy-MM-dd') - the_field['validation_messages']['minmax'] = field.validation_message('date minmax', self, word("You need to enter a date between %s and %s."), parameters=(docassemble.base.util.format_date(self.question.interview.options['default date min'], format='medium'), docassemble.base.util.format_date(self.question.interview.options['default date max'], format='medium'))) + the_field['min'] = format_date(self.question.interview.options['default date min'], format='yyyy-MM-dd') + the_field['max'] = format_date(self.question.interview.options['default date max'], format='yyyy-MM-dd') + the_field['validation_messages']['minmax'] = field.validation_message('date minmax', self, word("You need to enter a date between %s and %s."), parameters=(format_date(self.question.interview.options['default date min'], format='medium'), format_date(self.question.interview.options['default date max'], format='medium'))) elif 'max' not in was_defined and 'default date max' in self.question.interview.options: - the_field['max'] = docassemble.base.util.format_date(self.question.interview.options['default date max'], format='yyyy-MM-dd') - the_field['validation_messages']['max'] = field.validation_message('date max', self, word("You need to enter a date on or before %s."), parameters=tuple([docassemble.base.util.format_date(self.question.interview.options['default date max'], format='medium')])) + the_field['max'] = format_date(self.question.interview.options['default date max'], format='yyyy-MM-dd') + the_field['validation_messages']['max'] = field.validation_message('date max', self, word("You need to enter a date on or before %s."), parameters=tuple([format_date(self.question.interview.options['default date max'], format='medium')])) elif 'min' not in was_defined and 'default date min' in self.question.interview.options: - the_field['min'] = docassemble.base.util.format_date(self.question.interview.options['default date min'], format='yyyy-MM-dd') - the_field['validation_messages']['min'] = field.validation_message('date min', self, word("You need to enter a date on or after %s."), parameters=tuple([docassemble.base.util.format_date(self.question.interview.options['default date min'], format='medium')])) + the_field['min'] = format_date(self.question.interview.options['default date min'], format='yyyy-MM-dd') + the_field['validation_messages']['min'] = field.validation_message('date min', self, word("You need to enter a date on or after %s."), parameters=tuple([format_date(self.question.interview.options['default date min'], format='medium')])) if field.datatype == 'time': the_field['validation_messages']['time'] = field.validation_message('time', self, word("You need to enter a valid time.")) if field.datatype in ['datetime', 'datetime-local']: @@ -1296,9 +1087,9 @@ def as_data(self, the_user_dict, encode=True): if 'label' in item: output['question'] += '

' + item['label'] + '

' if hasattr(field, 'aota'): - the_field['all_of_the_above'] = docassemble.base.filter.markdown_to_html(self.extras['aota'][field.number], do_terms=False, status=self, verbatim=(not encode)) + the_field['all_of_the_above'] = markdown_to_html(self.extras['aota'][field.number], do_terms=False, status=self, verbatim=not encode) if hasattr(field, 'nota'): - the_field['none_of_the_above'] = docassemble.base.filter.markdown_to_html(self.extras['nota'][field.number], do_terms=False, status=self, verbatim=(not encode)) + the_field['none_of_the_above'] = markdown_to_html(self.extras['nota'][field.number], do_terms=False, status=self, verbatim=not encode) if field.number in self.extras['ok']: the_field['active'] = self.extras['ok'][field.number] else: @@ -1335,7 +1126,7 @@ def as_data(self, the_user_dict, encode=True): if the_expression: the_field['show_if_js'] = {'expression': the_expression, 'vars': field.extras['show_if_js']['vars'], 'sign': field.extras['show_if_js']['sign'], 'mode': field.extras['show_if_js']['mode']} if 'note' in self.extras and field.number in self.extras['note']: - the_field['note'] = docassemble.base.filter.markdown_to_html(self.extras['note'][field.number], status=self, verbatim=(not encode)) + the_field['note'] = markdown_to_html(self.extras['note'][field.number], status=self, verbatim=not encode) if 'html' in self.extras and field.number in self.extras['html']: the_field['html'] = self.extras['html'][field.number] if 'raw html' in self.extras and field.number in self.extras['raw html']: @@ -1345,21 +1136,21 @@ def as_data(self, the_user_dict, encode=True): if debug: output['question'] += '

' + the_field['hint'] + '

' if field.number in self.labels: - the_field['label'] = docassemble.base.filter.markdown_to_html(self.labels[field.number], trim=True, status=self, verbatim=(not encode)) + the_field['label'] = markdown_to_html(self.labels[field.number], trim=True, status=self, verbatim=not encode) if debug: output['question'] += '

' + the_field['label'] + '

' if field.number in self.helptexts: - the_field['helptext'] = docassemble.base.filter.markdown_to_html(self.helptexts[field.number], status=self, verbatim=(not encode)) + the_field['helptext'] = markdown_to_html(self.helptexts[field.number], status=self, verbatim=not encode) if debug: output['question'] += '

' + the_field['helptext'] + '

' if self.question.question_type in ("yesno", "yesnomaybe"): - the_field['true_label'] = docassemble.base.filter.markdown_to_html(self.question.yes(), trim=True, do_terms=False, status=self, verbatim=(not encode)) - the_field['false_label'] = docassemble.base.filter.markdown_to_html(self.question.no(), trim=True, do_terms=False, status=self, verbatim=(not encode)) + the_field['true_label'] = markdown_to_html(self.question.yes(), trim=True, do_terms=False, status=self, verbatim=not encode) + the_field['false_label'] = markdown_to_html(self.question.no(), trim=True, do_terms=False, status=self, verbatim=not encode) if debug: output['question'] += '

' + the_field['true_label'] + '

' output['question'] += '

' + the_field['false_label'] + '

' if self.question.question_type == 'yesnomaybe': - the_field['maybe_label'] = docassemble.base.filter.markdown_to_html(self.question.maybe(), trim=True, do_terms=False, status=self, verbatim=(not encode)) + the_field['maybe_label'] = markdown_to_html(self.question.maybe(), trim=True, do_terms=False, status=self, verbatim=not encode) if debug: output['question'] += '

' + the_field['maybe_label'] + '

' result['fields'].append(the_field) @@ -1381,7 +1172,7 @@ def as_data(self, the_user_dict, encode=True): for question_type in ('question', 'help'): if question_type not in output: continue - phrase = docassemble.base.functions.server.to_text('
' + output[question_type] + '
') + phrase = to_text('
' + output[question_type] + '
') if (not phrase) or len(phrase) < 10: phrase = "The sky is blue." phrase = re.sub(r'[^A-Za-z 0-9\.\,\?\#\!\%\&\(\)]', r' ', phrase) @@ -1449,7 +1240,7 @@ def icon_url(self, name): return None if the_image.attribution is not None: self.attributions.add(the_image.attribution) - url = docassemble.base.functions.server.url_finder(str(the_image.package) + ':' + str(the_image.filename)) + url = url_finder(str(the_image.package) + ':' + str(the_image.filename)) return url def get_choices_data(self, field, defaultvalue, the_user_dict, encode=True): @@ -1460,9 +1251,9 @@ def get_choices_data(self, field, defaultvalue, the_user_dict, encode=True): if self.question.question_type == "multiple_choice": pairlist = list(self.selectcompute[field.number]) for pair in pairlist: - item = {'label': docassemble.base.filter.markdown_to_html(pair['label'], trim=True, do_terms=False, status=self, verbatim=encode), 'value': pair['key']} + item = {'label': markdown_to_html(pair['label'], trim=True, do_terms=False, status=self, verbatim=encode), 'value': pair['key']} if 'help' in pair: - item['help'] = docassemble.base.filter.markdown_to_html(pair['help'].rstrip(), trim=True, do_terms=False, status=self, verbatim=encode) + item['help'] = markdown_to_html(pair['help'].rstrip(), trim=True, do_terms=False, status=self, verbatim=encode) for standard_key in ('default', 'css class', 'color', 'group'): if standard_key in pair: item[standard_key] = pair[standard_key] @@ -1486,7 +1277,7 @@ def get_choices_data(self, field, defaultvalue, the_user_dict, encode=True): pairlist = [] if field.datatype in ('object_multiselect', 'object_checkboxes'): for pair in pairlist: - item = {'label': docassemble.base.filter.markdown_to_html(pair['label'], trim=True, do_terms=False, status=self, verbatim=encode), 'value': from_safeid(pair['key'])} + item = {'label': markdown_to_html(pair['label'], trim=True, do_terms=False, status=self, verbatim=encode), 'value': from_safeid(pair['key'])} if ('default' in pair and pair['default']) or (defaultvalue is not None and isinstance(defaultvalue, (list, set)) and str(pair['key']) in defaultvalue) or (isinstance(defaultvalue, dict) and str(pair['key']) in defaultvalue and defaultvalue[str(pair['key'])]) or (isinstance(defaultvalue, (str, int, bool, float)) and str(pair['key']) == str(defaultvalue)): item['selected'] = True for standard_key in ('help', 'css class', 'color'): @@ -1499,7 +1290,7 @@ def get_choices_data(self, field, defaultvalue, the_user_dict, encode=True): choice_list.append(item) elif field.datatype in ('object', 'object_radio'): for pair in pairlist: - item = {'label': docassemble.base.filter.markdown_to_html(pair['label'], trim=True, do_terms=False, status=self, verbatim=encode), 'value': from_safeid(pair['key'])} + item = {'label': markdown_to_html(pair['label'], trim=True, do_terms=False, status=self, verbatim=encode), 'value': from_safeid(pair['key'])} if ('default' in pair and pair['default']) or (defaultvalue is not None and isinstance(defaultvalue, (str, int, bool, float)) and str(pair['key']) == str(defaultvalue)): item['selected'] = True if 'default' in pair: @@ -1514,7 +1305,7 @@ def get_choices_data(self, field, defaultvalue, the_user_dict, encode=True): choice_list.append(item) elif field.datatype in ('multiselect', 'checkboxes'): for pair in pairlist: - item = {'label': docassemble.base.filter.markdown_to_html(pair['label'], trim=True, do_terms=False, status=self, verbatim=encode), 'variable_name': saveas + "[" + repr(pair['key']) + "]", 'value': True} + item = {'label': markdown_to_html(pair['label'], trim=True, do_terms=False, status=self, verbatim=encode), 'variable_name': saveas + "[" + repr(pair['key']) + "]", 'value': True} if encode: item['variable_name_encoded'] = safeid(saveas + "[" + repr(pair['key']) + "]") if ('default' in pair and pair['default']) or (defaultvalue is not None and isinstance(defaultvalue, (list, set)) and str(pair['key']) in defaultvalue) or (isinstance(defaultvalue, dict) and str(pair['key']) in defaultvalue and defaultvalue[str(pair['key'])]) or (isinstance(defaultvalue, (str, int, bool, float)) and str(pair['key']) == str(defaultvalue)): @@ -1529,7 +1320,7 @@ def get_choices_data(self, field, defaultvalue, the_user_dict, encode=True): choice_list.append(item) else: for pair in pairlist: - item = {'label': docassemble.base.filter.markdown_to_html(pair['label'], trim=True, do_terms=False, status=self, verbatim=encode), 'value': pair['key']} + item = {'label': markdown_to_html(pair['label'], trim=True, do_terms=False, status=self, verbatim=encode), 'value': pair['key']} if ('default' in pair and pair['default']) or (defaultvalue is not None and isinstance(defaultvalue, (str, int, bool, float)) and str(pair['key']) == str(defaultvalue)): item['selected'] = True choice_list.append(item) @@ -1538,17 +1329,17 @@ def get_choices_data(self, field, defaultvalue, the_user_dict, encode=True): formatted_item = word("All of the above") else: formatted_item = self.extras['aota'][field.number] - choice_list.append({'label': docassemble.base.filter.markdown_to_html(formatted_item, trim=True, do_terms=False, status=self, verbatim=encode)}) + choice_list.append({'label': markdown_to_html(formatted_item, trim=True, do_terms=False, status=self, verbatim=encode)}) if hasattr(field, 'nota') and self.extras['nota'][field.number] is not False: if self.extras['nota'][field.number] is True: formatted_item = word("None of the above") else: formatted_item = self.extras['nota'][field.number] - choice_list.append({'label': docassemble.base.filter.markdown_to_html(formatted_item, trim=True, do_terms=False, status=self, verbatim=encode)}) + choice_list.append({'label': markdown_to_html(formatted_item, trim=True, do_terms=False, status=self, verbatim=encode)}) else: indexno = 0 for choice in self.selectcompute[field.number]: - item = {'label': docassemble.base.filter.markdown_to_html(choice['label'], trim=True, do_terms=False, status=self, verbatim=encode), 'variable_name': '_internal["answers"][' + repr(question.extended_question_name(the_user_dict)) + ']', 'value': indexno} + item = {'label': markdown_to_html(choice['label'], trim=True, do_terms=False, status=self, verbatim=encode), 'variable_name': '_internal["answers"][' + repr(question.extended_question_name(the_user_dict)) + ']', 'value': indexno} if encode: item['variable_name_encoded'] = safeid('_internal["answers"][' + repr(question.extended_question_name(the_user_dict)) + ']') if 'image' in choice: @@ -1629,7 +1420,7 @@ def __init__(self, x, question=None, translate=True): def text(self, the_user_dict): if len(self.other_lang) > 0: - target_lang = docassemble.base.functions.get_language() + target_lang = get_language() if self.language != target_lang and target_lang in self.other_lang: if self.uses_mako: return self.other_lang[target_lang][1].render(**the_user_dict) @@ -1869,7 +1660,7 @@ def recursive_eval_textobject(target, the_user_dict, question, tpl, skip_undefin text = '' else: text = target.text(the_user_dict) - return docassemble.base.file_docx.transform_for_docx(text) + return transform_for_docx(text) raise DAError("recursive_eval_textobject: expected a TextObject, but found a " + str(type(target))) @@ -1917,25 +1708,6 @@ def recursive_eval_textobject_or_primitive(target, the_user_dict): raise DAError("recursive_eval_textobject_or_primitive: expected a TextObject, but found a " + str(type(target))) -def fix_quotes(match): - instring = match.group(1) - n = len(instring) - output = '' - i = 0 - while i < n: - if instring[i] == '\u201c' or instring[i] == '\u201d': - output += '"' - elif instring[i] == '\u2018' or instring[i] == '\u2019': - output += "'" - elif instring[i] == '&' and i + 4 < n and instring[i:i+5] == '&': - output += '&' - i += 4 - else: - output += instring[i] - i += 1 - return output - - def docx_variable_fix(variable): variable = re.sub(r'\\', '', variable) variable = re.sub(r'^([A-Za-z\_][A-Za-z\_0-9]*).*', r'\1', variable) @@ -1950,7 +1722,7 @@ class FileInPackage: def __init__(self, fileref, area, package): if area == 'template' and not isinstance(fileref, dict): - docassemble.base.functions.package_template_filename(fileref, package=package) + package_template_filename(fileref, package=package) self.fileref = fileref if isinstance(self.fileref, dict): self.is_code = True @@ -1991,9 +1763,9 @@ def path(self, the_user_dict=None): raise DAError("FileInPackage: error downloading " + str(the_file_ref) + ": " + str(err)) the_file_ref = temp_template_file.name if not str(the_file_ref).startswith('/'): - the_file_ref = docassemble.base.functions.package_template_filename(str(the_file_ref), package=self.package) + the_file_ref = package_template_filename(str(the_file_ref), package=self.package) return the_file_ref - return docassemble.base.functions.package_template_filename(self.fileref, package=self.package) + return package_template_filename(self.fileref, package=self.package) return None def paths(self, the_user_dict=None): @@ -2025,11 +1797,11 @@ def paths(self, the_user_dict=None): else: result.append(the_file_ref) else: - result.append(docassemble.base.functions.package_template_filename(self.fileref, package=self.package)) + result.append(package_template_filename(self.fileref, package=self.package)) final_result = [] for the_file_ref in result: if not str(the_file_ref).startswith('/'): - final_result.append(docassemble.base.functions.package_template_filename(str(the_file_ref), package=self.package)) + final_result.append(package_template_filename(str(the_file_ref), package=self.package)) else: final_result.append(the_file_ref) return final_result @@ -2043,7 +1815,7 @@ def __init__(self, fileref, question): self.question = question def path(self): - info = docassemble.base.functions.server.file_finder(self.fileref, question=self.question) + info = file_finder(self.fileref, question=self.question) if 'fullpath' in info and info['fullpath']: return info['fullpath'] raise DAError("Could not find the file " + str(self.fileref)) @@ -2171,7 +1943,7 @@ def __init__(self, orig_data, caller, **kwargs): raise DASourceError("This block is missing a 'question' directive." + self.idebug(data)) if self.interview.debug: for key in data: - if key not in ('features', 'scan for variables', 'only sets', 'question', 'code', 'event', 'translations', 'default language', 'on change', 'sections', 'progressive', 'auto open', 'section', 'machine learning storage', 'language', 'prevent going back', 'back button', 'usedefs', 'continue button label', 'continue button color', 'resume button label', 'resume button color', 'back button label', 'corner back button label', 'skip undefined', 'list collect', 'mandatory', 'attachment options', 'script', 'css', 'initial', 'default role', 'command', 'objects from file', 'use objects', 'data', 'variable name', 'data from code', 'objects', 'id', 'ga id', 'segment id', 'segment', 'supersedes', 'order', 'image sets', 'images', 'def', 'mako', 'interview help', 'default screen parts', 'default validation messages', 'generic object', 'generic list object', 'comment', 'metadata', 'modules', 'reset', 'imports', 'terms', 'auto terms', 'role', 'include', 'action buttons', 'if', 'validation code', 'require', 'orelse', 'attachment', 'attachments', 'attachment code', 'attachments code', 'allow emailing', 'allow downloading', 'email subject', 'email body', 'email template', 'email address default', 'progress', 'zip filename', 'action', 'backgroundresponse', 'response', 'binaryresponse', 'all_variables', 'response filename', 'content type', 'redirect url', 'null response', 'sleep', 'include_internal', 'css class', 'table css class', 'response code', 'subquestion', 'reload', 'help', 'audio', 'video', 'decoration', 'signature', 'under', 'pre', 'post', 'right', 'check in', 'yesno', 'noyes', 'yesnomaybe', 'noyesmaybe', 'sets', 'event', 'choices', 'buttons', 'dropdown', 'combobox', 'field', 'shuffle', 'review', 'need', 'depends on', 'target', 'table', 'rows', 'columns', 'require gathered', 'allow reordering', 'edit', 'delete buttons', 'confirm', 'read only', 'edit header', 'confirm', 'show if empty', 'template', 'content file', 'content', 'subject', 'reconsider', 'undefine', 'continue button field', 'fields', 'indent', 'url', 'default', 'datatype', 'extras', 'allowed to set', 'show incomplete', 'not available label', 'required', 'always include editable files', 'question metadata', 'include attachment notice', 'include download tab', 'describe file types', 'manual attachment list', 'breadcrumb', 'tabular', 'hide continue button', 'disable continue button', 'pen color', 'gathered', 'sort key', 'filter'): + if key not in ('features', 'scan for variables', 'only sets', 'question', 'code', 'event', 'translations', 'default language', 'on change', 'sections', 'progressive', 'auto open', 'section', 'machine learning storage', 'language', 'prevent going back', 'back button', 'usedefs', 'continue button label', 'continue button color', 'resume button label', 'resume button color', 'back button label', 'corner back button label', 'skip undefined', 'list collect', 'mandatory', 'attachment options', 'script', 'css', 'initial', 'default role', 'command', 'objects from file', 'use objects', 'data', 'variable name', 'data from code', 'objects', 'id', 'ga id', 'segment id', 'segment', 'supersedes', 'order', 'image sets', 'images', 'def', 'mako', 'interview help', 'default screen parts', 'default validation messages', 'generic object', 'generic list object', 'comment', 'metadata', 'modules', 'reset', 'imports', 'terms', 'auto terms', 'role', 'include', 'action buttons', 'if', 'validation code', 'require', 'orelse', 'attachment', 'attachments', 'attachment code', 'attachments code', 'allow emailing', 'allow downloading', 'email subject', 'email body', 'email template', 'email address default', 'progress', 'zip filename', 'action', 'backgroundresponse', 'response', 'binaryresponse', 'all_variables', 'response filename', 'content type', 'redirect url', 'null response', 'sleep', 'include_internal', 'css class', 'table css class', 'response code', 'subquestion', 'reload', 'help', 'audio', 'video', 'decoration', 'signature', 'under', 'pre', 'post', 'right', 'check in', 'yesno', 'noyes', 'yesnomaybe', 'noyesmaybe', 'sets', 'event', 'choices', 'buttons', 'dropdown', 'combobox', 'field', 'shuffle', 'review', 'need', 'depends on', 'target', 'table', 'rows', 'columns', 'require gathered', 'allow reordering', 'edit', 'delete buttons', 'confirm', 'read only', 'edit header', 'confirm', 'show if empty', 'template', 'content file', 'content', 'subject', 'reconsider', 'undefine', 'continue button field', 'fields', 'indent', 'url', 'default', 'datatype', 'extras', 'allowed to set', 'show incomplete', 'not available label', 'required', 'always include editable files', 'question metadata', 'include attachment notice', 'include download tab', 'describe file types', 'manual attachment list', 'breadcrumb', 'tabular', 'hide continue button', 'disable continue button', 'pen color', 'gathered', 'sort key', 'sort reverse', 'filter', 'flattened checkbox label', 'flattened checkbox unselected label'): logmessage("Ignoring unknown dictionary key '" + key + "'." + self.idebug(data)) if 'features' in data: should_append = False @@ -2262,7 +2034,7 @@ def __init__(self, orig_data, caller, **kwargs): data['features']['custom datatypes to load'] = [data['features']['custom datatypes to load']] if isinstance(data['features']['custom datatypes to load'], list): for item in data['features']['custom datatypes to load']: - if isinstance(item, str) and item not in standard_types and item in docassemble.base.functions.custom_types: + if isinstance(item, str) and item not in standard_types and item in custom_types: self.interview.custom_data_types.add(item) if 'checkin interval' in data['features']: if not isinstance(data['features']['checkin interval'], int): @@ -2302,7 +2074,7 @@ def __init__(self, orig_data, caller, **kwargs): if not isinstance(data['features'][key], str): raise DASourceError("A features section " + key + " entry must be plain text." + self.idebug(data)) try: - self.interview.options[key] = dateutil.parser.parse(data['features'][key]).astimezone(zoneinfo.ZoneInfo(docassemble.base.functions.get_default_timezone())) + self.interview.options[key] = dateutil.parser.parse(data['features'][key]).astimezone(zoneinfo.ZoneInfo(get_default_timezone())) except: raise DASourceError("The " + key + " in features did not contain a valid date." + self.idebug(data)) if 'field' in data and not ('yesno' in data or 'noyes' in data or 'yesnomaybe' in data or 'noyesmaybe' in data or 'buttons' in data or 'choices' in data or 'dropdown' in data or 'combobox' in data): @@ -2351,7 +2123,7 @@ def __init__(self, orig_data, caller, **kwargs): for item in tr_todo: self.interview.translations.append(item) if item.endswith(".xlsx"): - the_xlsx_file = docassemble.base.functions.package_data_filename(item) + the_xlsx_file = package_data_filename(item) if not os.path.isfile(the_xlsx_file): raise DAError("The translations file " + the_xlsx_file + " could not be found") import pandas # pylint: disable=import-outside-toplevel @@ -2368,7 +2140,7 @@ def __init__(self, orig_data, caller, **kwargs): self.interview.translation_dict[df['orig_text'][indexno]][df['orig_lang'][indexno]] = {} self.interview.translation_dict[df['orig_text'][indexno]][df['orig_lang'][indexno]][df['tr_lang'][indexno]] = df['tr_text'][indexno] elif item.endswith(".xlf") or item.endswith(".xliff"): - the_xlf_file = docassemble.base.functions.package_data_filename(item) + the_xlf_file = package_data_filename(item) if not os.path.isfile(the_xlf_file): continue tree = ET.parse(the_xlf_file) @@ -3158,19 +2930,19 @@ def __init__(self, orig_data, caller, **kwargs): if isinstance(data['include'], str): data['include'] = [data['include']] if isinstance(data['include'], list): - for questionPath in data['include']: + for question_path in data['include']: try: - if ':' in questionPath: - self.interview.read_from(interview_source_from_string(questionPath, interview_source=self.interview.source, parent_source=self.from_source)) + if ':' in question_path: + self.interview.read_from(interview_source_from_string(question_path, interview_source=self.interview.source, parent_source=self.from_source)) else: - new_source = self.from_source.append(questionPath) + new_source = self.from_source.append(question_path) if new_source is None: - new_source = interview_source_from_string('docassemble.base:data/questions/' + re.sub(r'^data/questions/', '', questionPath), interview_source=self.interview.source, parent_source=self.from_source) + new_source = interview_source_from_string('docassemble.base:data/questions/' + re.sub(r'^data/questions/', '', question_path), interview_source=self.interview.source, parent_source=self.from_source) if new_source is None: - raise DANotFoundError('Question file ' + questionPath + ' not found') + raise DANotFoundError('Question file ' + question_path + ' not found') self.interview.read_from(new_source) except DANotFoundError: - raise DASourceError('An include section could not find the file ' + str(questionPath) + '.' + self.idebug(data)) + raise DASourceError('An include section could not find the file ' + str(question_path) + '.' + self.idebug(data)) else: raise DASourceError("An include section must be organized as a list." + self.idebug(data)) if 'action buttons' in data: @@ -3365,7 +3137,7 @@ def __init__(self, orig_data, caller, **kwargs): if hasattr(data['response filename'], 'mimetype') and data['response filename'].mimetype: self.content_type = TextObject(data['response filename'].mimetype) else: - info = docassemble.base.functions.server.file_finder(data['response filename'], question=self) + info = file_finder(data['response filename'], question=self) if 'fullpath' in info and info['fullpath']: self.response_file = FileOnServer(data['response filename'], self) # info['fullpath'] else: @@ -3642,7 +3414,7 @@ def __init__(self, orig_data, caller, **kwargs): field_data['saveas'] = data['field'] if 'datatype' in data and 'type' not in field_data: field_data['type'] = data['datatype'] - if data['datatype'] not in standard_types and data['datatype'] in docassemble.base.functions.custom_types: + if data['datatype'] not in standard_types and data['datatype'] in custom_types: self.interview.custom_data_types.add(data['datatype']) elif is_boolean(field_data): field_data['type'] = 'boolean' @@ -3879,7 +3651,7 @@ def __init__(self, orig_data, caller, **kwargs): for content_file in data['content file']: if not isinstance(content_file, str): raise DASourceError('A content file must be specified as text, as a list of text filenames, or as a dictionary with code as the key' + self.idebug(data)) - file_to_read = docassemble.base.functions.package_template_filename(content_file, package=self.package) + file_to_read = package_template_filename(content_file, package=self.package) # if file_to_read is not None and get_mimetype(file_to_read) != 'text/markdown': # raise DASourceError('The content file ' + str(data['content file']) + ' is not a markdown file ' + str(file_to_read) + self.idebug(data)) if file_to_read is not None and os.path.isfile(file_to_read) and os.access(file_to_read, os.R_OK): @@ -3976,7 +3748,7 @@ def __init__(self, orig_data, caller, **kwargs): raise DASourceError("The fields must be written in the form of a list." + self.idebug(data)) field_number = 0 for field in data['fields']: - docassemble.base.functions.this_thread.misc['current_field'] = field_number + this_thread.misc['current_field'] = field_number if not isinstance(field, dict): raise DASourceError("Each individual field in a list of fields must be expressed as a dictionary item, e.g., ' - Fruit: user.favorite_fruit'." + self.idebug(data)) manual_keys = set() @@ -4008,7 +3780,7 @@ def __init__(self, orig_data, caller, **kwargs): field['datatype'] = 'text' if field['datatype'] in ('object', 'object_radio', 'multiselect', 'object_multiselect', 'checkboxes', 'object_checkboxes') and not ('choices' in field or 'code' in field): raise DASourceError("A multiple choice field must refer to a list of choices." + self.idebug(data)) - if field['datatype'] in docassemble.base.functions.custom_types and field['datatype'] not in standard_types: + if field['datatype'] in custom_types and field['datatype'] not in standard_types: custom_data_type = True self.interview.custom_data_types.add(field['datatype']) if 'input type' in field: @@ -4027,8 +3799,8 @@ def __init__(self, orig_data, caller, **kwargs): field_info['extras'] = {'fields_code': compile(field['code'], '', 'eval')} self.fields.append(Field(field_info)) field_number += 1 - if 'current_field' in docassemble.base.functions.this_thread.misc: - del docassemble.base.functions.this_thread.misc['current_field'] + if 'current_field' in this_thread.misc: + del this_thread.misc['current_field'] continue if 'object labeler' in field and ('datatype' not in field or not field['datatype'].startswith('object')): raise DASourceError("An object labeler can only be used with an object data type." + self.idebug(data)) @@ -4038,14 +3810,14 @@ def __init__(self, orig_data, caller, **kwargs): if key == 'default' and 'datatype' in field and field['datatype'] in ('object', 'object_radio', 'object_multiselect', 'object_checkboxes'): continue if custom_data_type: - if key in docassemble.base.functions.custom_types[field['datatype']]['parameters']: + if key in custom_types[field['datatype']]['parameters']: if 'extras' not in field_info: field_info['extras'] = {} if 'custom_parameters' not in field_info['extras']: field_info['extras']['custom_parameters'] = {} field_info['extras']['custom_parameters'][key] = field[key] continue - if key in docassemble.base.functions.custom_types[field['datatype']]['code_parameters']: + if key in custom_types[field['datatype']]['code_parameters']: if 'extras' not in field_info: field_info['extras'] = {} if 'custom_parameters_code' not in field_info['extras']: @@ -4053,7 +3825,7 @@ def __init__(self, orig_data, caller, **kwargs): field_info['extras']['custom_parameters_code'][key] = {'compute': compile(str(field[key]), '', 'eval'), 'sourcecode': str(field[key])} self.find_fields_in(field[key]) continue - if key in docassemble.base.functions.custom_types[field['datatype']]['mako_parameters']: + if key in custom_types[field['datatype']]['mako_parameters']: if 'extras' not in field_info: field_info['extras'] = {} if 'custom_parameters_mako' not in field_info['extras']: @@ -4648,8 +4420,8 @@ def __init__(self, orig_data, caller, **kwargs): else: raise DASourceError("A field was listed without indicating a label or a variable name, and the field was not a note or raw HTML." + self.idebug(data) + " and field_info was " + repr(field_info)) field_number += 1 - if 'current_field' in docassemble.base.functions.this_thread.misc: - del docassemble.base.functions.this_thread.misc['current_field'] + if 'current_field' in this_thread.misc: + del this_thread.misc['current_field'] if 'review' in data: self.question_type = 'review' if self.is_mandatory and 'continue button field' not in data: @@ -5016,18 +4788,18 @@ def exec_setup(self, is_generic, the_x, iterators, the_user_dict): for indexno, item in enumerate(iterators): exec(list_of_indices[indexno] + " = " + item, the_user_dict) for the_field in [substitute_vars(item, is_generic, the_x, iterators) for item in self.undefine]: - docassemble.base.functions.undefine(the_field) + undefine(the_field) if len(self.reconsider) > 0: - docassemble.base.functions.reconsider(*[substitute_vars(item, is_generic, the_x, iterators) for item in self.reconsider]) + reconsider(*[substitute_vars(item, is_generic, the_x, iterators) for item in self.reconsider]) if self.need is not None: for need_code in self.need: eval(need_code, the_user_dict) def exec_setup_mandatory(self, the_user_dict): for the_field in self.undefine: - docassemble.base.functions.undefine(the_field) + undefine(the_field) if len(self.reconsider) > 0: - docassemble.base.functions.reconsider(*self.reconsider) + reconsider(*self.reconsider) if self.need is not None: for need_code in self.need: eval(need_code, the_user_dict) @@ -5075,7 +4847,7 @@ def recursive_dataobject(self, target): return TextObject(str(target), question=self) def find_fields_in(self, code): - myvisitor = myvisitnode() + myvisitor = MyVisitNode() t = ast.parse(str(code)) myvisitor.visit(t) predefines = set(globals().keys()) | set(locals().keys()) @@ -5150,7 +4922,7 @@ def process_attachment(self, orig_target): if 'name' not in target: target['name'] = word("Document") if 'filename' not in target: - # target['filename'] = docassemble.base.functions.space_to_underscore(target['name']) + # target['filename'] = space_to_underscore(target['name']) target['filename'] = '' if 'description' not in target: target['description'] = '' @@ -5271,7 +5043,7 @@ def process_attachment(self, orig_target): for content_file in target['content file']: if not isinstance(content_file, str): raise DASourceError('A content file must be specified as text, a list of text filenames, or a dictionary where the one key is code' + self.idebug(target)) - file_to_read = docassemble.base.functions.package_template_filename(content_file, package=self.package) + file_to_read = package_template_filename(content_file, package=self.package) if file_to_read is not None and os.path.isfile(file_to_read) and os.access(file_to_read, os.R_OK): with open(file_to_read, 'r', encoding='utf-8') as the_file: target['content'] += the_file.read() @@ -5355,7 +5127,7 @@ def process_attachment(self, orig_target): if len(template_files) == 1: the_docx_path = template_files[0] else: - the_docx_path = docassemble.base.file_docx.concatenate_files(template_files) + the_docx_path = concatenate_files(template_files) try: docx_template = DocxTemplate(the_docx_path) docx_template.render_init() @@ -5516,6 +5288,10 @@ def process_attachment(self, orig_target): raise DASourceError('Unknown data type in attachment pdftk.' + self.idebug(target)) if 'rendering font' in target and target['rendering font']: options['rendering_font'] = TextObject(str(target['rendering font']), question=self) + if 'flattened checkbox label' in target and target['flattened checkbox label'] is not None: + options['flattened_checkbox_label'] = TextObject(str(target['flattened checkbox label']), question=self) + if 'flattened checkbox unselected label' in target and target['flattened checkbox unselected label'] is not None: + options['flattened_checkbox_unselected_label'] = TextObject(str(target['flattened checkbox unselected label']), question=self) if 'tagged pdf' in target: if isinstance(target['tagged pdf'], bool): options['tagged_pdf'] = target['tagged pdf'] @@ -5561,7 +5337,7 @@ def get_question_for_field_with_sub_fields(self, field, user_dict): if not isinstance(field_list, list): raise DAError("A code directive that defines items in fields must return a list") new_interview_source = InterviewSourceString(content='') - new_interview = new_interview_source.get_interview() + new_interview = Interview(source=new_interview_source) reproduce_basics(self.interview, new_interview) return Question({'question': 'n/a', 'fields': field_list}, new_interview, source=new_interview_source, package=self.package) @@ -5587,7 +5363,7 @@ def get_fields_and_sub_fields(self, user_dict): def ask(self, user_dict, old_user_dict, the_x, iterators, sought, orig_sought, process_list_collect=True, test_for_objects=True): # logmessage("ask: orig_sought is " + str(orig_sought) + " and q is " + self.name) - docassemble.base.functions.this_thread.current_question = self + this_thread.current_question = self if the_x != 'None': exec("x = " + the_x, user_dict) if len(iterators) > 0: @@ -5598,18 +5374,18 @@ def ask(self, user_dict, old_user_dict, the_x, iterators, sought, orig_sought, p for need_code in self.need: eval(need_code, user_dict) for the_field in self.undefine: - docassemble.base.functions.undefine(the_field) + undefine(the_field) if len(self.reconsider) > 0: - docassemble.base.functions.reconsider(*[substitute_vars(item, self.is_generic, the_x, iterators) for item in self.reconsider]) + reconsider(*[substitute_vars(item, self.is_generic, the_x, iterators) for item in self.reconsider]) if self.section: - docassemble.base.functions.this_thread.current_section = self.section.text(user_dict).strip() + this_thread.current_section = self.section.text(user_dict).strip() question_text = self.content.text(user_dict).rstrip() if self.breadcrumb is not None: breadcrumb = self.breadcrumb.text(user_dict).rstrip() else: breadcrumb = None try: - user_dict['_internal']['event_stack'][docassemble.base.functions.this_thread.current_info['user']['session_uid']][0]['breadcrumb'] = question_text if breadcrumb is None else breadcrumb + user_dict['_internal']['event_stack'][this_thread.current_info['user']['session_uid']][0]['breadcrumb'] = question_text if breadcrumb is None else breadcrumb except: pass # logmessage("Asking " + str(question_text)) @@ -5654,7 +5430,7 @@ def ask(self, user_dict, old_user_dict, the_x, iterators, sought, orig_sought, p if forget_prior: arguments = {'_action': action, '_arguments': arguments} action = '_da_priority_action' - action = docassemble.base.functions.url_action(action, **arguments) + action = url_action(action, **arguments) color = item['color'].text(user_dict).strip() if item['target'] is not None: target = item['target'].text(user_dict).strip() @@ -5714,7 +5490,7 @@ def ask(self, user_dict, old_user_dict, the_x, iterators, sought, orig_sought, p if forget_prior: arguments = {'_action': action, '_arguments': arguments} action = '_da_priority_action' - action = docassemble.base.functions.url_action(action, **arguments) + action = url_action(action, **arguments) label = button['label'] extras['action_buttons'].append({'action': action, 'label': label, 'color': color, 'icon': icon, 'placement': placement, 'css_class': css_class, 'target': target}) for item in extras['action_buttons']: @@ -5797,7 +5573,7 @@ def ask(self, user_dict, old_user_dict, the_x, iterators, sought, orig_sought, p if key in ('pre', 'post', 'footer', 'submit', 'exit link', 'exit label', 'exit url', 'full', 'logo', 'short logo', 'title', 'subtitle', 'tab title', 'short title', 'title url', 'title url opens in other window', 'navigation bar html') and (key + ' text') not in extras: extras[key + ' text'] = val if len(self.terms) > 0: - lang = docassemble.base.functions.get_language() + lang = get_language() extras['terms'] = {} for termitem, definition in self.terms.items(): if lang in definition['alt_terms']: @@ -5805,7 +5581,7 @@ def ask(self, user_dict, old_user_dict, the_x, iterators, sought, orig_sought, p else: extras['terms'][termitem] = {'definition': definition['definition'].text(user_dict)} if len(self.autoterms) > 0: - lang = docassemble.base.functions.get_language() + lang = get_language() extras['autoterms'] = {} for termitem, definition in self.autoterms.items(): if lang in definition['alt_terms']: @@ -6050,7 +5826,7 @@ def ask(self, user_dict, old_user_dict, the_x, iterators, sought, orig_sought, p else: extras['allow_emailing'] = eval(self.allow_emailing, user_dict) if hasattr(self, 'zip_filename'): - extras['zip_filename'] = docassemble.base.functions.single_paragraph(self.zip_filename.text(user_dict)) + extras['zip_filename'] = single_paragraph(self.zip_filename.text(user_dict)) if hasattr(self, 'ga_id'): extras['ga_id'] = self.ga_id.text(user_dict) if hasattr(self, 'segment') and 'id' in self.segment: @@ -6076,7 +5852,7 @@ def ask(self, user_dict, old_user_dict, the_x, iterators, sought, orig_sought, p skip_undefined = True extras['ok'] = {} for field in self.fields: - docassemble.base.functions.this_thread.misc['current_field'] = field.number + this_thread.misc['current_field'] = field.number extras['ok'][field.number] = False if hasattr(field, 'saveas_code'): failed = False @@ -6170,8 +5946,8 @@ def ask(self, user_dict, old_user_dict, the_x, iterators, sought, orig_sought, p else: labels[field.number] = field.label.text(user_dict) extras['ok'][field.number] = True - if 'current_field' in docassemble.base.functions.this_thread.misc: - del docassemble.base.functions.this_thread.misc['current_field'] + if 'current_field' in this_thread.misc: + del this_thread.misc['current_field'] else: if hasattr(self, 'list_collect') and process_list_collect and eval(self.list_collect, user_dict): fields_to_scan = self.get_fields_and_sub_fields(user_dict) @@ -6250,7 +6026,7 @@ def ask(self, user_dict, old_user_dict, the_x, iterators, sought, orig_sought, p new_iterators[iterator_index] = str(list_indexno) except IndexError: raise DAException("list collect question needs iterator " + extras['list_iterator'] + " but it was asked in a context where there is no " + extras['list_iterator']) - ask_result = self.ask(user_dict, old_user_dict, the_x, new_iterators, sought, orig_sought, process_list_collect=False, test_for_objects=(list_indexno < length_to_use)) + ask_result = self.ask(user_dict, old_user_dict, the_x, new_iterators, sought, orig_sought, process_list_collect=False, test_for_objects=list_indexno < length_to_use) if hasattr(self, 'list_collect_label'): extras['list_message'][list_indexno] = self.list_collect_label.text(user_dict) else: @@ -6304,7 +6080,7 @@ def ask(self, user_dict, old_user_dict, the_x, iterators, sought, orig_sought, p for field in self.fields: if hasattr(field, 'inputtype') and field.inputtype in ('combobox', 'datalist'): only_empty_fields_exist = False - docassemble.base.functions.this_thread.misc['current_field'] = field.number + this_thread.misc['current_field'] = field.number if hasattr(field, 'has_code') and field.has_code: # standalone multiple-choice questions selectcompute[field.number] = [] @@ -6505,11 +6281,11 @@ def ask(self, user_dict, old_user_dict, the_x, iterators, sought, orig_sought, p if complications.search(var) or var not in user_dict: eval(var, user_dict) raise CodeExecute(commands_to_run, self) - if 'current_field' in docassemble.base.functions.this_thread.misc: - del docassemble.base.functions.this_thread.misc['current_field'] + if 'current_field' in this_thread.misc: + del this_thread.misc['current_field'] extras['ok'] = {} for field in self.fields: - docassemble.base.functions.this_thread.misc['current_field'] = field.number + this_thread.misc['current_field'] = field.number if hasattr(field, 'showif_code'): result = eval(field.showif_code, user_dict) if hasattr(field, 'extras') and 'show_if_sign_code' in field.extras and field.extras['show_if_sign_code'] == 0: @@ -6795,8 +6571,8 @@ def ask(self, user_dict, old_user_dict, the_x, iterators, sought, orig_sought, p hints[field.number] = field.hint.text(user_dict) if hasattr(field, 'helptext'): helptexts[field.number] = field.helptext.text(user_dict) - if 'current_field' in docassemble.base.functions.this_thread.misc: - del docassemble.base.functions.this_thread.misc['current_field'] + if 'current_field' in this_thread.misc: + del this_thread.misc['current_field'] if len(self.attachments) > 0 or self.compute_attachment is not None: if hasattr(self, 'email_default'): the_email_address = self.email_default.text(user_dict).strip() @@ -6805,7 +6581,7 @@ def ask(self, user_dict, old_user_dict, the_x, iterators, sought, orig_sought, p if hasattr(self, 'email_subject'): extras['email_subject'] = re.sub(r'[\n\r]+', ' ', self.email_subject.text(user_dict).strip()) if hasattr(self, 'email_body'): - extras['email_html'] = '' + docassemble.base.filter.markdown_to_html(self.email_body.text(user_dict), status=docassemble.base.functions.this_thread.interview_status, question=self, external=True) + '' + extras['email_html'] = '' + markdown_to_html(self.email_body.text(user_dict), status=this_thread.interview_status, question=self, external=True) + '' extras['email_body'] = BeautifulSoup(extras['email_html'], "html.parser").get_text('\n') if hasattr(self, 'email_template') and ('email_subject' not in extras or 'email_html' not in extras): template = eval(self.email_template, user_dict) @@ -6824,7 +6600,7 @@ def ask(self, user_dict, old_user_dict, the_x, iterators, sought, orig_sought, p for field in self.fields: if field.number in extras['ok'] and not extras['ok'][field.number]: continue - docassemble.base.functions.this_thread.misc['current_field'] = field.number + this_thread.misc['current_field'] = field.number if hasattr(field, 'saveas'): # m = re.match(r'(.*)\.[^\.]+', from_safeid(field.saveas)) # if m and m.group(1) != 'x': @@ -6836,8 +6612,8 @@ def ask(self, user_dict, old_user_dict, the_x, iterators, sought, orig_sought, p assumed_objects.add(parse_result['objects'][-1]) if len(parse_result['bracket_objects']) > 0: assumed_objects.add(parse_result['bracket_objects'][-1]) - if 'current_field' in docassemble.base.functions.this_thread.misc: - del docassemble.base.functions.this_thread.misc['current_field'] + if 'current_field' in this_thread.misc: + del this_thread.misc['current_field'] for var in assumed_objects: if complications.search(var) or var not in user_dict: eval(var, user_dict) @@ -6861,7 +6637,7 @@ def ask(self, user_dict, old_user_dict, the_x, iterators, sought, orig_sought, p if self.question_type == 'review' and sought is not None and not hasattr(self, 'review_saveas'): if 'event_stack' not in user_dict['_internal']: user_dict['_internal']['event_stack'] = {} - session_uid = docassemble.base.functions.this_thread.current_info['user']['session_uid'] + session_uid = this_thread.current_info['user']['session_uid'] if session_uid not in user_dict['_internal']['event_stack']: user_dict['_internal']['event_stack'][session_uid] = [] already_there = False @@ -7094,11 +6870,11 @@ def finalize_attachment(self, attachment, result, the_user_dict): return result except: pass - docassemble.base.functions.this_thread.misc['redact'] = bool(result['redact']) - docassemble.base.functions.this_thread.misc['attachment_info'] = {k: result[k] for k in ('name', 'filename', 'description', 'update_references', 'convert_to_pdf_a', 'convert_to_tagged_pdf') if k in result} + this_thread.misc['redact'] = bool(result['redact']) + this_thread.misc['attachment_info'] = {k: result[k] for k in ('name', 'filename', 'description', 'update_references', 'convert_to_pdf_a', 'convert_to_tagged_pdf') if k in result} if 'language' in attachment['options']: - old_language = docassemble.base.functions.get_language() - docassemble.base.functions.set_language(attachment['options']['language']) + old_language = get_language() + set_language(attachment['options']['language']) else: old_language = None try: @@ -7107,13 +6883,13 @@ def finalize_attachment(self, attachment, result, the_user_dict): with tempfile.NamedTemporaryFile(prefix="datemp", mode="wb", suffix=result['raw'], delete=False) as the_temp: with open(the_temp.name, 'w', encoding='utf-8') as the_file: the_file.write(result['markdown'][doc_format].lstrip("\n")) - result['file'][doc_format], result['extension'][doc_format], result['mimetype'][doc_format] = docassemble.base.functions.server.save_numbered_file(result['filename'] + result['raw'], the_temp.name, yaml_file_name=self.interview.source.path) # pylint: disable=assignment-from-none,unpacking-non-sequence + result['file'][doc_format], result['extension'][doc_format], result['mimetype'][doc_format] = save_numbered_file(result['filename'] + result['raw'], the_temp.name, yaml_file_name=self.interview.source.path) # pylint: disable=assignment-from-none,unpacking-non-sequence result['content'][doc_format] = result['markdown'][doc_format].lstrip("\n") elif doc_format == 'md': with tempfile.NamedTemporaryFile(prefix="datemp", mode="wb", suffix='md', delete=False) as the_temp: with open(the_temp.name, 'w', encoding='utf-8') as the_file: the_file.write(result['markdown'][doc_format].lstrip("\n")) - result['file'][doc_format], result['extension'][doc_format], result['mimetype'][doc_format] = docassemble.base.functions.server.save_numbered_file(result['filename'] + '.md', the_temp.name, yaml_file_name=self.interview.source.path) # pylint: disable=assignment-from-none,unpacking-non-sequence + result['file'][doc_format], result['extension'][doc_format], result['mimetype'][doc_format] = save_numbered_file(result['filename'] + '.md', the_temp.name, yaml_file_name=self.interview.source.path) # pylint: disable=assignment-from-none,unpacking-non-sequence result['content'][doc_format] = result['markdown'][doc_format].lstrip("\n") elif doc_format in ('pdf', 'rtf', 'rtf to docx', 'tex', 'docx'): if 'fields' in attachment['options']: @@ -7122,33 +6898,33 @@ def finalize_attachment(self, attachment, result, the_user_dict): default_export_value = attachment['options']['checkbox_export_value'].text(the_user_dict).strip() else: default_export_value = None - docassemble.base.functions.set_context('pdf') + set_context('pdf') the_template_path = attachment['options']['pdf_template_file'].path(the_user_dict=the_user_dict) if the_template_path is None: raise DASourceError("pdf template file " + attachment['options']['pdf_template_file'].original_reference() + " not found") - the_pdf_file = docassemble.base.pdftk.fill_template(the_template_path, data_strings=result['data_strings'], images=result['images'], editable=result['editable'], pdfa=result['convert_to_pdf_a'], use_pdftk=result['use_pdftk'], password=result['password'], owner_password=result['owner_password'], template_password=result['template_password'], default_export_value=default_export_value, replacement_font=result['rendering_font']) - result['file'][doc_format], result['extension'][doc_format], result['mimetype'][doc_format] = docassemble.base.functions.server.save_numbered_file(result['filename'] + '.' + extension_of_doc_format.get(doc_format, doc_format), the_pdf_file, yaml_file_name=self.interview.source.path) # pylint: disable=assignment-from-none,unpacking-non-sequence + the_pdf_file = fill_template(the_template_path, data_strings=result['data_strings'], images=result['images'], editable=result['editable'], pdfa=result['convert_to_pdf_a'], use_pdftk=result['use_pdftk'], password=result['password'], owner_password=result['owner_password'], template_password=result['template_password'], default_export_value=default_export_value, replacement_font=result['rendering_font'], flattened_checkbox_label=result.get('flattened_checkbox_label', None), flattened_checkbox_unselected_label=result.get('flattened_checkbox_unselected_label', None)) + result['file'][doc_format], result['extension'][doc_format], result['mimetype'][doc_format] = save_numbered_file(result['filename'] + '.' + extension_of_doc_format.get(doc_format, doc_format), the_pdf_file, yaml_file_name=self.interview.source.path) # pylint: disable=assignment-from-none,unpacking-non-sequence for key in ('images', 'data_strings', 'convert_to_pdf_a', 'use_pdftk', 'convert_to_tagged_pdf', 'password', 'owner_password', 'template_password', 'update_references', 'permissions', 'rendering_font'): if key in result: del result[key] - docassemble.base.functions.reset_context() + reset_context() elif (doc_format == 'docx' or (doc_format == 'pdf' and 'docx' not in result['formats_to_use'])) and 'docx_template_file' in attachment['options']: # logmessage("field_data is " + repr(result['field_data'])) if result['template'].current_rendering_part is None: result['template'].current_rendering_part = result['template'].docx._part - docassemble.base.functions.set_context('docx', template=result['template']) - docassemble.base.functions.this_thread.misc['docx_subdocs'] = [] - docassemble.base.functions.this_thread.misc['auto jinja filter'] = [] + set_context('docx', template=result['template']) + this_thread.misc['docx_subdocs'] = [] + this_thread.misc['auto jinja filter'] = [] if 'auto jinja filter' in self.interview.options: for item in self.interview.options['auto jinja filter']: - docassemble.base.functions.this_thread.misc['auto jinja filter'].append(eval(item, the_user_dict)) + this_thread.misc['auto jinja filter'].append(eval(item, the_user_dict)) try: the_template = result['template'] template_loop_count = 0 while True: # Rerender if there's a subdoc using include_docx_template - old_count = docassemble.base.functions.this_thread.misc.get('docx_include_count', 0) + old_count = this_thread.misc.get('docx_include_count', 0) the_template.render(result['field_data'], jinja_env=custom_jinja_env(skip_undefined=attachment['options']['skip_undefined'])) - if docassemble.base.functions.this_thread.misc.get('docx_include_count', 0) > old_count and template_loop_count < 10: + if this_thread.misc.get('docx_include_count', 0) > old_count and template_loop_count < 10: # There's another template included with tempfile.NamedTemporaryFile(prefix="datemp", mode="wb", suffix=".docx", delete=False) as new_template_file: the_template.save(new_template_file.name) # Save and refresh the template @@ -7162,12 +6938,12 @@ def finalize_attachment(self, attachment, result, the_user_dict): the_template.da_hyperlink_style = 'InternetLink' else: the_template.da_hyperlink_style = None - docassemble.base.functions.this_thread.misc['docx_template'] = the_template + this_thread.misc['docx_template'] = the_template template_loop_count += 1 else: break # Copy over images, etc from subdoc to master template - # subdocs = docassemble.base.functions.this_thread.misc.get('docx_subdocs', []) # Get the subdoc file list + # subdocs = this_thread.misc.get('docx_subdocs', []) # Get the subdoc file list # the_template_docx = the_template.docx except TemplateError as the_error: @@ -7179,19 +6955,19 @@ def finalize_attachment(self, attachment, result, the_user_dict): the_error.filename = ', '.join(docx_paths) # logmessage("TemplateError:\n" + traceback.format_exc()) raise the_error - docassemble.base.functions.reset_context() + reset_context() with tempfile.NamedTemporaryFile(prefix="datemp", mode="wb", suffix=".docx", delete=False) as docx_file: the_template.save(docx_file.name) - docassemble.base.file_docx.fix_docx(docx_file.name) + fix_docx(docx_file.name) if result['update_references']: - docassemble.base.pandoc.update_references(docx_file.name) # does this update refs twice? + update_references(docx_file.name) # does this update refs twice? if 'pdf' in result['formats_to_use']: with tempfile.NamedTemporaryFile(prefix="datemp", mode="wb", suffix=".pdf", delete=False) as pdf_file: - if not docassemble.base.pandoc.word_to_pdf(docx_file.name, 'docx', pdf_file.name, pdfa=result['convert_to_pdf_a'], password=result['password'], update_refs=result['update_references'], tagged=result['convert_to_tagged_pdf'], filename=result['filename']): + if not word_to_pdf(docx_file.name, 'docx', pdf_file.name, pdfa=result['convert_to_pdf_a'], password=result['password'], update_refs=result['update_references'], tagged=result['convert_to_tagged_pdf'], filename=result['filename']): raise DAException('Failure to convert DOCX to PDF') - result['file']['pdf'], result['extension']['pdf'], result['mimetype']['pdf'] = docassemble.base.functions.server.save_numbered_file(result['filename'] + '.pdf', pdf_file.name, yaml_file_name=self.interview.source.path) # pylint: disable=assignment-from-none,unpacking-non-sequence + result['file']['pdf'], result['extension']['pdf'], result['mimetype']['pdf'] = save_numbered_file(result['filename'] + '.pdf', pdf_file.name, yaml_file_name=self.interview.source.path) # pylint: disable=assignment-from-none,unpacking-non-sequence if 'docx' in result['formats_to_use']: - result['file']['docx'], result['extension']['docx'], result['mimetype']['docx'] = docassemble.base.functions.server.save_numbered_file(result['filename'] + '.docx', docx_file.name, yaml_file_name=self.interview.source.path) # pylint: disable=assignment-from-none,unpacking-non-sequence + result['file']['docx'], result['extension']['docx'], result['mimetype']['docx'] = save_numbered_file(result['filename'] + '.docx', docx_file.name, yaml_file_name=self.interview.source.path) # pylint: disable=assignment-from-none,unpacking-non-sequence for key in ['template', 'field_data', 'images', 'data_strings', 'convert_to_pdf_a', 'convert_to_tagged_pdf', 'password', 'owner_password', 'template_password', 'update_references', 'permissions', 'rendering_font']: if key in result: del result[key] @@ -7224,10 +7000,10 @@ def finalize_attachment(self, attachment, result, the_user_dict): converter.template_file = self.interview.attachment_options['template_file'].path(the_user_dict=the_user_dict) converter.metadata = result['metadata'] converter.convert(self) - result['file'][doc_format], result['extension'][doc_format], result['mimetype'][doc_format] = docassemble.base.functions.server.save_numbered_file(result['filename'] + '.' + extension_of_doc_format.get(doc_format, doc_format), converter.output_filename, yaml_file_name=self.interview.source.path) # pylint: disable=assignment-from-none,unpacking-non-sequence + result['file'][doc_format], result['extension'][doc_format], result['mimetype'][doc_format] = save_numbered_file(result['filename'] + '.' + extension_of_doc_format.get(doc_format, doc_format), converter.output_filename, yaml_file_name=self.interview.source.path) # pylint: disable=assignment-from-none,unpacking-non-sequence result['content'][doc_format] = result['markdown'][doc_format] elif doc_format == 'html': - result['content'][doc_format] = docassemble.base.filter.markdown_to_html(result['markdown'][doc_format], use_pandoc=True, question=self) + result['content'][doc_format] = markdown_to_html(result['markdown'][doc_format], use_pandoc=True, question=self) elif doc_format == 'md': result['content'][doc_format] = result['markdown'][doc_format] if 'manual' in result: @@ -7258,13 +7034,13 @@ def finalize_attachment(self, attachment, result, the_user_dict): if attachment['variable_name']: the_string = "from docassemble.base.util import DAFile, DAFileCollection" exec(the_string, the_user_dict) - variable_name = substitute_vars_from_user_dict(docassemble.base.functions.intrinsic_name_of(attachment['variable_name'], the_user_dict=the_user_dict), the_user_dict, is_generic=self.is_generic) + variable_name = substitute_vars_from_user_dict(intrinsic_name_of(attachment['variable_name'], the_user_dict=the_user_dict), the_user_dict, is_generic=self.is_generic) the_string = variable_name + " = DAFileCollection(" + repr(variable_name) + ")" exec(the_string, the_user_dict) the_name = attachment['name'].text(the_user_dict).strip() - the_filename = docassemble.base.functions.secure_filename_unicode_ok(attachment['filename'].text(the_user_dict).strip()) + the_filename = secure_filename_unicode_ok(attachment['filename'].text(the_user_dict).strip()) if the_filename == '': - the_filename = docassemble.base.functions.secure_filename_unicode_ok(docassemble.base.functions.space_to_underscore(the_name)) + the_filename = secure_filename_unicode_ok(space_to_underscore(the_name)) the_user_dict['_attachment_info'] = {'name': the_name, 'filename': the_filename, 'description': attachment['description'].text(the_user_dict), 'valid_formats': result['valid_formats'], 'formats': result['formats_to_use'], 'attachment': {'name': attachment['question_name'], 'number': attachment['indexno']}, 'extension': result.get('extension', {}), 'mimetype': result.get('mimetype', {}), 'content': result.get('content', {}), 'markdown': result.get('markdown', {}), 'metadata': result.get('metadata', {}), 'convert_to_pdf_a': result.get('convert_to_pdf_a', False), 'convert_to_tagged_pdf': result.get('convert_to_tagged_pdf', False), 'orig_variable_name': result.get('orig_variable_name', None), 'raw': result['raw'], 'permissions': result.get('permissions', None)} if len(manual_files) > 0: the_user_dict['_attachment_info']['manual_formats'] = result['manual_formats'] @@ -7273,7 +7049,7 @@ def finalize_attachment(self, attachment, result, the_user_dict): for doc_format in result['file']: variable_string = variable_name + '.' + extension_of_doc_format.get(doc_format, doc_format) # filename = result['filename'] + '.' + doc_format - # file_number, extension, mimetype = docassemble.base.functions.server.save_numbered_file(filename, result['file'][doc_format], yaml_file_name=self.interview.source.path) # pylint: disable=assignment-from-none,unpacking-non-sequence + # file_number, extension, mimetype = save_numbered_file(filename, result['file'][doc_format], yaml_file_name=self.interview.source.path) # pylint: disable=assignment-from-none,unpacking-non-sequence if result['file'][doc_format] is None: raise DAError("Could not save numbered file") if doc_format in manual_files: @@ -7325,20 +7101,20 @@ def finalize_attachment(self, attachment, result, the_user_dict): exec(the_string, the_user_dict) except: if old_language is not None: - docassemble.base.functions.set_language(old_language) - docassemble.base.functions.this_thread.misc.pop('redact', None) - docassemble.base.functions.this_thread.misc.pop('attachment_info', None) + set_language(old_language) + this_thread.misc.pop('redact', None) + this_thread.misc.pop('attachment_info', None) raise if old_language is not None: - docassemble.base.functions.set_language(old_language) - docassemble.base.functions.this_thread.misc.pop('redact', None) - docassemble.base.functions.this_thread.misc.pop('attachment_info', None) + set_language(old_language) + this_thread.misc.pop('redact', None) + this_thread.misc.pop('attachment_info', None) return result def prepare_attachment(self, attachment, the_user_dict): if 'language' in attachment['options']: - old_language = docassemble.base.functions.get_language() - docassemble.base.functions.set_language(attachment['options']['language']) + old_language = get_language() + set_language(attachment['options']['language']) else: old_language = None if isinstance(attachment['valid_formats'], CodeType): @@ -7353,9 +7129,9 @@ def prepare_attachment(self, attachment, the_user_dict): try: the_name = attachment['name'].text(the_user_dict).strip() the_filename = attachment['filename'].text(the_user_dict).strip() - the_filename = docassemble.base.functions.secure_filename_unicode_ok(the_filename) + the_filename = secure_filename_unicode_ok(the_filename) if the_filename == '': - the_filename = docassemble.base.functions.secure_filename_unicode_ok(docassemble.base.functions.space_to_underscore(the_name)) + the_filename = secure_filename_unicode_ok(space_to_underscore(the_name)) result = {'name': the_name, 'filename': the_filename, 'description': attachment['description'].text(the_user_dict), 'valid_formats': copy.deepcopy(valid_formats)} actual_extension = attachment['raw'] if attachment['content'] is None and 'content file code' in attachment['options']: @@ -7379,7 +7155,7 @@ def prepare_attachment(self, attachment, the_user_dict): raise DASourceError("prepare_attachment: error downloading " + str(the_filename) + ": " + str(err)) the_filename = temp_template_file.name else: - the_filename = docassemble.base.functions.package_template_filename(the_filename, package=self.package) + the_filename = package_template_filename(the_filename, package=self.package) else: the_filename = None if the_filename is None or not os.path.isfile(the_filename): @@ -7401,8 +7177,16 @@ def prepare_attachment(self, attachment, the_user_dict): result['editable'] = eval(attachment['options']['editable'], the_user_dict) else: result['editable'] = True - docassemble.base.functions.this_thread.misc['redact'] = bool(result['redact']) - docassemble.base.functions.this_thread.misc['attachment_info'] = {k: result[k] for k in ('name', 'filename', 'description', 'update_references', 'convert_to_pdf_a', 'convert_to_tagged_pdf') if k in result} + if 'flattened_checkbox_label' in attachment['options']: + result['flattened_checkbox_label'] = attachment['options']['flattened_checkbox_label'].text(the_user_dict) + else: + result['flattened_checkbox_label'] = None + if 'flattened_checkbox_unselected_label' in attachment['options']: + result['flattened_checkbox_unselected_label'] = attachment['options']['flattened_checkbox_unselected_label'].text(the_user_dict) + else: + result['flattened_checkbox_unselected_label'] = None + this_thread.misc['redact'] = bool(result['redact']) + this_thread.misc['attachment_info'] = {k: result[k] for k in ('name', 'filename', 'description', 'update_references', 'convert_to_pdf_a', 'convert_to_tagged_pdf') if k in result} result['markdown'] = {} result['content'] = {} result['extension'] = {} @@ -7411,7 +7195,7 @@ def prepare_attachment(self, attachment, the_user_dict): if attachment['raw']: if '.' in the_filename: m = re.search(r'(.*)(\..*)', the_filename) - result['filename'] = docassemble.base.functions.secure_filename_unicode_ok(m.group(1)) + result['filename'] = secure_filename_unicode_ok(m.group(1)) actual_extension = m.group(2) result['raw'] = actual_extension result['formats_to_use'] = ['raw'] @@ -7525,7 +7309,7 @@ def prepare_attachment(self, attachment, the_user_dict): if len(docx_paths) == 1: docx_path = docx_paths[0] else: - docx_path = docassemble.base.file_docx.concatenate_files(docx_paths) + docx_path = concatenate_files(docx_paths) result['template'] = DocxTemplate(docx_path) result['template'].render_init() if result['hyperlink_style'] and result['hyperlink_style'] in result['template'].docx.styles: @@ -7538,7 +7322,7 @@ def prepare_attachment(self, attachment, the_user_dict): result['template'].da_hyperlink_style = None if result['template'].current_rendering_part is None: result['template'].current_rendering_part = result['template'].docx._part - docassemble.base.functions.set_context('docx', template=result['template']) + set_context('docx', template=result['template']) if isinstance(attachment['options']['fields'], str): result['field_data'] = the_user_dict else: @@ -7574,7 +7358,7 @@ def prepare_attachment(self, attachment, the_user_dict): elif isinstance(val, RawValue): result['field_data'][key] = val.value else: - result['field_data'][key] = docassemble.base.file_docx.transform_for_docx(val) + result['field_data'][key] = transform_for_docx(val) else: raise DAError("code in an attachment returned something other than a dictionary") if 'raw code dict' in attachment['options']: @@ -7604,10 +7388,10 @@ def prepare_attachment(self, attachment, the_user_dict): elif isinstance(val, RawValue): result['field_data'][varname] = val.value else: - result['field_data'][varname] = docassemble.base.file_docx.transform_for_docx(val) - docassemble.base.functions.reset_context() + result['field_data'][varname] = transform_for_docx(val) + reset_context() elif doc_format == 'pdf' and 'fields' in attachment['options'] and 'pdf_template_file' in attachment['options']: - docassemble.base.functions.set_context('pdf') + set_context('pdf') result['data_strings'] = [] result['images'] = [] if isinstance(attachment['options']['fields'], dict): @@ -7635,9 +7419,9 @@ def prepare_attachment(self, attachment, the_user_dict): m = re.search(r'\[FILE ([^\]]+)\]', answer) if m: file_reference = re.sub(r'[ ,].*', '', m.group(1)) - file_info = docassemble.base.functions.server.file_finder(file_reference, question=self) + file_info = file_finder(file_reference, question=self) if 'path' in file_info and 'extension' in file_info: - docassemble.base.filter.convert_svg_to_png(file_info) + convert_svg_to_png(file_info) result['images'].append((key, file_info)) else: m = re.search(r'\[QR ([^\]]+)\]', answer) @@ -7677,9 +7461,9 @@ def prepare_attachment(self, attachment, the_user_dict): m = re.search(r'\[FILE ([^\]]+)\]', val) if m: file_reference = re.sub(r'[ ,].*', '', m.group(1)) - file_info = docassemble.base.functions.server.file_finder(file_reference, question=self) + file_info = file_finder(file_reference, question=self) if 'path' in file_info and 'extension' in file_info: - docassemble.base.filter.convert_svg_to_png(file_info) + convert_svg_to_png(file_info) result['images'].append((key, file_info)) else: m = re.search(r'\[QR ([^\]]+)\]', val) @@ -7720,9 +7504,9 @@ def prepare_attachment(self, attachment, the_user_dict): m = re.search(r'\[FILE ([^\]]+)\]', val) if m: file_reference = re.sub(r'[ ,].*', '', m.group(1)) - file_info = docassemble.base.functions.server.file_finder(file_reference, question=self) + file_info = file_finder(file_reference, question=self) if 'path' in file_info and 'extension' in file_info: - docassemble.base.filter.convert_svg_to_png(file_info) + convert_svg_to_png(file_info) result['images'].append((key, file_info)) else: m = re.search(r'\[QR ([^\]]+)\]', val) @@ -7761,9 +7545,9 @@ def prepare_attachment(self, attachment, the_user_dict): m = re.search(r'\[FILE ([^\]]+)\]', val) if m: file_reference = re.sub(r'[ ,].*', '', m.group(1)) - file_info = docassemble.base.functions.server.file_finder(file_reference, question=self) + file_info = file_finder(file_reference, question=self) if 'path' in file_info and 'extension' in file_info: - docassemble.base.filter.convert_svg_to_png(file_info) + convert_svg_to_png(file_info) result['images'].append((key, file_info)) else: m = re.search(r'\[QR ([^\]]+)\]', val) @@ -7774,24 +7558,24 @@ def prepare_attachment(self, attachment, the_user_dict): result['images'].append((key, {'fullpath': the_image.name})) else: result['data_strings'].append((key, val)) - docassemble.base.functions.reset_context() + reset_context() elif doc_format in ('raw', 'md'): - docassemble.base.functions.set_context(doc_format) + set_context(doc_format) the_markdown = the_content.text(the_user_dict) result['markdown'][doc_format] = the_markdown - docassemble.base.functions.reset_context() + reset_context() else: modified_metadata = {"syslang": get_language()} if result['convert_to_tagged_pdf']: modified_metadata['taggedpdf'] = "true" for key, data in result['metadata'].items(): if re.search(r'Footer|Header', key) and 'Lines' not in key: - # modified_metadata[key] = docassemble.base.filter.metadata_filter(data, doc_format) + str('[END]') + # modified_metadata[key] = metadata_filter(data, doc_format) + str('[END]') modified_metadata[key] = data + str('[END]') else: modified_metadata[key] = data the_markdown = '---\n' + altyaml.dump_to_string(modified_metadata) + "\n...\n" - docassemble.base.functions.set_context('pandoc ' + doc_format) + set_context('pandoc ' + doc_format) the_markdown += the_content.text(the_user_dict) # logmessage("Markdown is:\n" + repr(the_markdown) + "END") do_not_scan_for_emojis = bool(re.search(r'\[NO_EMOJIS\]', the_markdown)) @@ -7800,27 +7584,27 @@ def prepare_attachment(self, attachment, the_user_dict): elif emoji_match.search(the_markdown) and len(self.interview.images) > 0: the_markdown = emoji_match.sub(emoji_matcher_insert(self), the_markdown) result['markdown'][doc_format] = the_markdown - docassemble.base.functions.reset_context() + reset_context() elif doc_format in ['html']: - docassemble.base.functions.set_context('html') + set_context('html') result['markdown'][doc_format] = the_content.text(the_user_dict) do_not_scan_for_emojis = bool(re.search(r'\[NO_EMOJIS\]', result['markdown'][doc_format])) if do_not_scan_for_emojis: result['markdown'][doc_format] = re.sub(r'\[NO_EMOJIS\]\s*', r'', result['markdown'][doc_format]) elif emoji_match.search(result['markdown'][doc_format]) and len(self.interview.images) > 0: result['markdown'][doc_format] = emoji_match.sub(emoji_matcher_html(self), result['markdown'][doc_format]) - docassemble.base.functions.reset_context() + reset_context() # logmessage("output was:\n" + repr(result['content'][doc_format])) except: if old_language is not None: - docassemble.base.functions.set_language(old_language) - docassemble.base.functions.this_thread.misc.pop('redact', None) - docassemble.base.functions.this_thread.misc.pop('attachment_info', None) + set_language(old_language) + this_thread.misc.pop('redact', None) + this_thread.misc.pop('attachment_info', None) raise if old_language is not None: - docassemble.base.functions.set_language(old_language) - docassemble.base.functions.this_thread.misc.pop('redact', None) - docassemble.base.functions.this_thread.misc.pop('attachment_info', None) + set_language(old_language) + this_thread.misc.pop('redact', None) + this_thread.misc.pop('attachment_info', None) if 'manual' in attachment['options']: result['manual'] = {extension: eval(expression, the_user_dict) for extension, expression in attachment['options']['manual'].items()} if 'manual code' in attachment['options']: @@ -7904,36 +7688,11 @@ def process_selections_manual(self, data): def emoji_matcher_insert(obj): - return (lambda x: docassemble.base.filter.emoji_insert(x.group(1), images=obj.interview.images)) + return (lambda x: emoji_insert(x.group(1), images=obj.interview.images)) def emoji_matcher_html(obj): - return (lambda x: docassemble.base.filter.emoji_html(x.group(1), images=obj.interview.images)) - - -def question_path_options(path): - n = 0 - while n < 3: - if n == 0: - yield docassemble.base.functions.package_question_filename(path) - elif n == 1: - yield docassemble.base.functions.standard_question_filename(path) - elif n == 2: - yield docassemble.base.functions.server.absolute_filename(path) - n += 1 - - -def interview_source_from_string(path, **kwargs): - if path is None: - raise DAError("Passed None to interview_source_from_string") - # logmessage("Trying to find " + path) - path = re.sub(r'(docassemble.playground[0-9]+[^:]*:)data/questions/(.*)', r'\1\2', path) - for the_filename in question_path_options(path): - if the_filename is not None: - new_source = InterviewSourceFile(filepath=the_filename, path=path) - if new_source.update(**kwargs): - return new_source - raise DANotFoundError("Interview " + str(path) + " not found") + return (lambda x: emoji_html(x.group(1), images=obj.interview.images)) def is_boolean(field_data): @@ -8016,7 +7775,7 @@ def illegal_variable_name(var): t = ast.parse(var) except: return True - detector = docassemble.base.astparser.detectIllegal() + detector = DetectIllegal() detector.visit(t) return detector.illegal @@ -8086,28 +7845,28 @@ def format_yaml_mark(mark, filename, line_number): return str(mark) -def format_yaml_errmess(errMess, filename, line_number): - if isinstance(errMess, ruamel.yaml.error.MarkedYAMLError): +def format_yaml_errmess(error_message, filename, line_number): + if isinstance(error_message, ruamel.yaml.error.MarkedYAMLError): lines = [] - if errMess.context is not None: - lines.append(errMess.context) - if errMess.context_mark is not None and ( - errMess.problem is None - or errMess.problem_mark is None - or errMess.context_mark.name != errMess.problem_mark.name - or errMess.context_mark.line != errMess.problem_mark.line - or errMess.context_mark.column != errMess.problem_mark.column + if error_message.context is not None: + lines.append(error_message.context) + if error_message.context_mark is not None and ( + error_message.problem is None + or error_message.problem_mark is None + or error_message.context_mark.name != error_message.problem_mark.name + or error_message.context_mark.line != error_message.problem_mark.line + or error_message.context_mark.column != error_message.problem_mark.column ): - lines.append(format_yaml_mark(errMess.context_mark, filename, line_number)) - if errMess.problem is not None: - lines.append(errMess.problem) - if errMess.problem_mark is not None: - lines.append(format_yaml_mark(errMess.problem_mark, filename, line_number)) - if errMess.note is not None and errMess.note: - note = textwrap.dedent(errMess.note) + lines.append(format_yaml_mark(error_message.context_mark, filename, line_number)) + if error_message.problem is not None: + lines.append(error_message.problem) + if error_message.problem_mark is not None: + lines.append(format_yaml_mark(error_message.problem_mark, filename, line_number)) + if error_message.note is not None and error_message.note: + note = textwrap.dedent(error_message.note) lines.append(note) return '\n'.join(lines) - return str(errMess) + return str(error_message) class Interview: @@ -8181,7 +7940,7 @@ def __init__(self, **kwargs): self.consolidated_metadata = {} self.issue = {} self.custom_data_types = set() - self.default_language = docassemble.base.functions.server.default_language + self.default_language = get_default_language() if 'source' in kwargs: self.read_from(kwargs['source']) self.cross_reference_dependencies() @@ -8335,9 +8094,9 @@ def get_bootstrap_theme(self): if self.bootstrap_theme is None: return None if not hasattr(self, 'bootstrap_theme_package'): - result = docassemble.base.functions.server.url_finder(self.bootstrap_theme, _package=self.source.package) + result = url_finder(self.bootstrap_theme, _package=self.source.package) else: - result = docassemble.base.functions.server.url_finder(self.bootstrap_theme, _package=self.bootstrap_theme_package) + result = url_finder(self.bootstrap_theme, _package=self.bootstrap_theme_package) return result def get_tags(self, the_user_dict): @@ -8552,39 +8311,39 @@ def read_from(self, source): if document is not None: question = Question(document, self, source=source, package=source_package, source_code=source_code, line_number=line_number) self.names_used.update(question.fields_used) - except BaseException as errMess: + except BaseException as err_mess: # logmessage(str(source_code)) try: - logmessage(f'Interview: error reading YAML file {source.path} in the block on line {line_number}\nDocument source code was:\n\n---\n{source_code.strip()}\n---\n\nError was:\n\n{format_yaml_errmess(errMess, source.path, line_number)}') + logmessage(f'Interview: error reading YAML file {source.path} in the block on line {line_number}\nDocument source code was:\n\n---\n{source_code.strip()}\n---\n\nError was:\n\n{format_yaml_errmess(err_mess, source.path, line_number)}') except: try: - logmessage(f'Interview: error reading YAML file {source.path} in the block on line {line_number}. Error was:\n\n{errMess}') + logmessage(f'Interview: error reading YAML file {source.path} in the block on line {line_number}. Error was:\n\n{err_mess}') except: - logmessage(f'Interview: error reading YAML file {source.path} in the block on line {line_number}. Error type was:\n\n' + errMess.__class__.__name__) + logmessage(f'Interview: error reading YAML file {source.path} in the block on line {line_number}. Error type was:\n\n' + err_mess.__class__.__name__) self.success = False else: try: document = safeyaml.load(source_code) - except BaseException as errMess: + except BaseException as err_mess: self.success = False try: - error_to_raise = DASourceError(f'Error reading YAML file {source.path} in the block on line {line_number}\n\nDocument source code was:\n\n---\n{source_code.strip()}\n---\n\nError was:\n\n{format_yaml_errmess(errMess, source.path, line_number)}') + error_to_raise = DASourceError(f'Error reading YAML file {source.path} in the block on line {line_number}\n\nDocument source code was:\n\n---\n{source_code.strip()}\n---\n\nError was:\n\n{format_yaml_errmess(err_mess, source.path, line_number)}') except: - error_to_raise = DASourceError(f'Error reading YAML file {source.path} in the block on line {line_number}\n\nDocument source code was:\n\n---\n{source_code.strip()}\n---\n\nError was:\n\n' + str(errMess.__class__.__name__)) + error_to_raise = DASourceError(f'Error reading YAML file {source.path} in the block on line {line_number}\n\nDocument source code was:\n\n---\n{source_code.strip()}\n---\n\nError was:\n\n' + str(err_mess.__class__.__name__)) raise error_to_raise if document is not None: try: question = Question(document, self, source=source, package=source_package, source_code=source_code, line_number=line_number) self.names_used.update(question.fields_used) - except SyntaxException as qError: + except SyntaxException as question_error: self.success = False - raise DASourceError(f"Syntax Exception: {qError}\n\nIn file {source.path} in the block on line {line_number} from package {source_package}:\n{source_code}") - except CompileException as qError: + raise DASourceError(f"Syntax Exception: {question_error}\n\nIn file {source.path} in the block on line {line_number} from package {source_package}:\n{source_code}") + except CompileException as question_error: self.success = False - raise DASourceError(f"Compile Exception: {qError}\n\nIn file {source.path} in the block on line {line_number} from package {source_package}:\n{source_code}") - except SyntaxError as qError: + raise DASourceError(f"Compile Exception: {question_error}\n\nIn file {source.path} in the block on line {line_number} from package {source_package}:\n{source_code}") + except SyntaxError as question_error: self.success = False - raise DASourceError(f"Syntax Error: {qError}\n\nIn file {source.path} in the block on line {line_number} from package {source_package}:\n{source_code}") + raise DASourceError(f"Syntax Error: {question_error}\n\nIn file {source.path} in the block on line {line_number} from package {source_package}:\n{source_code}") line_number += lines_in_code for ordering in self.id_orderings: if ordering['type'] == 'supersedes' and hasattr(ordering['question'], 'number'): @@ -8621,13 +8380,13 @@ def read_from(self, source): for metadata in self.metadata: if 'social' in metadata and isinstance(metadata['social'], dict): if 'image' in metadata['social'] and isinstance(metadata['social']['image'], str): - metadata['social']['image'] = docassemble.base.functions.server.url_finder(metadata['social']['image'], _package=metadata['_origin_package'], _external=True) + metadata['social']['image'] = url_finder(metadata['social']['image'], _package=metadata['_origin_package'], _external=True) if metadata['social']['image'] is None: logmessage("Invalid image reference in social meta tags") del metadata['social']['image'] for key, subkey in (('og', 'image'), ('twitter', 'image')): if key in metadata['social'] and isinstance(metadata['social'][key], dict) and subkey in metadata['social'][key] and isinstance(metadata['social'][key][subkey], str): - metadata['social'][key][subkey] = docassemble.base.functions.server.url_finder(metadata['social'][key][subkey], _package=metadata['_origin_package'], _external=True) + metadata['social'][key][subkey] = url_finder(metadata['social'][key][subkey], _package=metadata['_origin_package'], _external=True) if metadata['social'][key][subkey] is None: logmessage("Invalid image reference in social meta tags") del metadata['social'][key][subkey] @@ -8655,7 +8414,7 @@ def read_from(self, source): self.default_title[lang][title_abb] = str(val).strip() else: self.default_title['*'][title_abb] = str(metadata[title_name]).strip() - for lang, parts in docassemble.base.functions.server.main_page_parts.items(): + for lang, parts in get_main_page_parts().items(): if lang not in self.default_title: self.default_title[lang] = {} for title_name, title_abb in mapping: @@ -8758,14 +8517,14 @@ def assemble(self, user_dict, interview_status=None, old_user_dict=None, force_q # if interview_status.current_info['url'] is not None: # user_dict['_internal']['url'] = interview_status.current_info['url'] interview_status.set_tracker(user_dict['_internal']['tracker']) - # docassemble.base.functions.reset_local_variables() + # reset_local_variables() interview_status.current_info.update({'default_role': self.default_role}) - docassemble.base.functions.this_thread.misc['reconsidered'] = set() - docassemble.base.functions.this_thread.current_package = self.source.package - docassemble.base.functions.this_thread.current_info = interview_status.current_info - docassemble.base.functions.this_thread.interview = self - docassemble.base.functions.this_thread.interview_status = interview_status - docassemble.base.functions.this_thread.internal = user_dict['_internal'] + this_thread.misc['reconsidered'] = set() + this_thread.current_package = self.source.package + this_thread.current_info = interview_status.current_info + this_thread.interview = self + this_thread.interview_status = interview_status + this_thread.internal = user_dict['_internal'] if user_dict['nav'].sections is None: user_dict['nav'].sections = self.sections if hasattr(self, 'sections_progressive'): @@ -8834,9 +8593,9 @@ def assemble(self, user_dict, interview_status=None, old_user_dict=None, force_q while True: number_loops += 1 if number_loops > self.loop_limit: - docassemble.base.functions.wrap_up() + wrap_up() raise DASourceError("There appears to be a circularity. Variables involved: " + ", ".join(variables_sought) + ".") - docassemble.base.functions.reset_gathering_mode() + reset_gathering_mode() if 'action' in interview_status.current_info: # logmessage("assemble: there is an action in the current_info: " + repr(interview_status.current_info['action'])) if interview_status.current_info['action'] in ('_da_list_remove', '_da_list_add', '_da_list_complete'): @@ -8862,7 +8621,7 @@ def assemble(self, user_dict, interview_status=None, old_user_dict=None, force_q if force_question is not None: if self.debug and question is not None: interview_status.seeking.append({'question': question, 'reason': 'multiple choice question', 'time': time.time()}) - docassemble.base.functions.this_thread.current_question = force_question + this_thread.current_question = force_question interview_status.populate(force_question.ask(user_dict, old_user_dict, 'None', [], None, None)) raise MandatoryQuestion() if not self.calls_process_action: @@ -8873,7 +8632,7 @@ def assemble(self, user_dict, interview_status=None, old_user_dict=None, force_q # logmessage("Running some initial code:\n\n" + question.sourcecode) if self.debug: interview_status.seeking.append({'question': question, 'reason': 'initial', 'time': time.time()}) - docassemble.base.functions.this_thread.current_question = question + this_thread.current_question = question question.exec_setup_mandatory(user_dict) exec_with_trap(question, user_dict) continue @@ -8891,9 +8650,9 @@ def assemble(self, user_dict, interview_status=None, old_user_dict=None, force_q gathered = eval(question.gathered, user_dict) thename = from_safeid(question.fields[0].saveas) if question.use_objects == 'objects': - user_dict['_DADATA'] = docassemble.base.util.objects_from_data(recursive_eval_dataobject(question.fields[0].data, user_dict), recursive=True, gathered=gathered, name=thename, package=question.package) + user_dict['_DADATA'] = objects_from_data(recursive_eval_dataobject(question.fields[0].data, user_dict), recursive=True, gathered=gathered, name=thename, package=question.package) elif question.use_objects: - user_dict['_DADATA'] = docassemble.base.util.objects_from_structure(recursive_eval_dataobject(question.fields[0].data, user_dict), root=thename, gathered=gathered) + user_dict['_DADATA'] = objects_from_structure(recursive_eval_dataobject(question.fields[0].data, user_dict), root=thename, gathered=gathered) else: user_dict['_DADATA'] = recursive_eval_dataobject(question.fields[0].data, user_dict) the_string = thename + ' = _DADATA' @@ -8910,9 +8669,9 @@ def assemble(self, user_dict, interview_status=None, old_user_dict=None, force_q gathered = eval(question.gathered, user_dict) thename = from_safeid(question.fields[0].saveas) if question.use_objects == 'objects': - user_dict['_DADATAFROMCODE'] = docassemble.base.util.objects_from_data(recursive_eval_data_from_code(question.fields[0].data, user_dict), recursive=True, gathered=gathered, name=thename, package=question.package) + user_dict['_DADATAFROMCODE'] = objects_from_data(recursive_eval_data_from_code(question.fields[0].data, user_dict), recursive=True, gathered=gathered, name=thename, package=question.package) elif question.use_objects: - user_dict['_DADATAFROMCODE'] = docassemble.base.util.objects_from_structure(recursive_eval_data_from_code(question.fields[0].data, user_dict), root=thename, gathered=gathered) + user_dict['_DADATAFROMCODE'] = objects_from_structure(recursive_eval_data_from_code(question.fields[0].data, user_dict), root=thename, gathered=gathered) else: user_dict['_DADATAFROMCODE'] = recursive_eval_data_from_code(question.fields[0].data, user_dict) the_string = thename + ' = _DADATAFROMCODE' @@ -8938,7 +8697,7 @@ def assemble(self, user_dict, interview_status=None, old_user_dict=None, force_q if self.debug: interview_status.seeking.append({'question': question, 'reason': 'objects', 'time': time.time()}) # logmessage("Going into objects") - docassemble.base.functions.this_thread.current_question = question + this_thread.current_question = question question.exec_setup_mandatory(user_dict) for keyvalue in question.objects: for variable in keyvalue: @@ -8959,7 +8718,7 @@ def assemble(self, user_dict, interview_status=None, old_user_dict=None, force_q interview_status.seeking.append({'question': question, 'reason': 'mandatory code', 'time': time.time()}) # logmessage("Running some code:\n\n" + question.sourcecode) # logmessage("Question name is " + question.name) - docassemble.base.functions.this_thread.current_question = question + this_thread.current_question = question question.exec_setup_mandatory(user_dict) exec_with_trap(question, user_dict) # logmessage("Code completed") @@ -8974,7 +8733,7 @@ def assemble(self, user_dict, interview_status=None, old_user_dict=None, force_q if self.debug and the_question is not question: interview_status.seeking.append({'question': the_question, 'reason': 'result of multiple choice', 'time': time.time()}) if the_question.question_type in ["code", "event_code"]: - docassemble.base.functions.this_thread.current_question = the_question + this_thread.current_question = the_question question.exec_setup_mandatory(user_dict) exec_with_trap(the_question, user_dict) interview_status.mark_tentative_as_answered(user_dict) @@ -8993,12 +8752,12 @@ def assemble(self, user_dict, interview_status=None, old_user_dict=None, force_q except ForcedReRun: continue except (NameError, UnboundLocalError, DAAttributeError, DAIndexError) as the_exception: - if 'pending_error' in docassemble.base.functions.this_thread.misc: - del docassemble.base.functions.this_thread.misc['pending_error'] + if 'pending_error' in this_thread.misc: + del this_thread.misc['pending_error'] # logmessage("Error in " + the_exception.__class__.__name__ + " is " + str(the_exception)) - if self.debug and docassemble.base.functions.this_thread.evaluation_context == 'docx': + if self.debug and this_thread.evaluation_context == 'docx': logmessage("NameError exception during document assembly: " + str(the_exception)) - docassemble.base.functions.reset_context() + reset_context() seeking_question = False if isinstance(the_exception, ForcedNameError): # logmessage("assemble: got a ForcedNameError for " + str(the_exception.name)) @@ -9023,10 +8782,10 @@ def assemble(self, user_dict, interview_status=None, old_user_dict=None, force_q user_dict['_internal']['event_stack'][session_uid] = new_items + user_dict['_internal']['event_stack'][session_uid] if exception_name.startswith('_da_'): continue - docassemble.base.functions.this_thread.misc['forgive_missing_question'] = [exception_name] + this_thread.misc['forgive_missing_question'] = [exception_name] if the_exception.arguments is not None: - docassemble.base.functions.this_thread.current_info.update({'action': exception_name, 'arguments': the_exception.arguments}) - missingVariable = exception_name + this_thread.current_info.update({'action': exception_name, 'arguments': the_exception.arguments}) + missing_variable = exception_name else: if type(the_exception) is NameError: # pylint: disable=unidiomatic-typecheck cl, exc, tb = sys.exc_info() @@ -9043,9 +8802,9 @@ def assemble(self, user_dict, interview_status=None, old_user_dict=None, force_q del exc del tb follow_mc = True - missingVariable = extract_missing_name(the_exception) - variables_sought.add(missingVariable) - question_result = self.askfor(missingVariable, user_dict, old_user_dict, interview_status, seeking=interview_status.seeking, follow_mc=follow_mc, seeking_question=seeking_question) + missing_variable = extract_missing_name(the_exception) + variables_sought.add(missing_variable) + question_result = self.askfor(missing_variable, user_dict, old_user_dict, interview_status, seeking=interview_status.seeking, follow_mc=follow_mc, seeking_question=seeking_question) if question_result['type'] in ('continue', 're_run'): continue if question_result['type'] == 'refresh': @@ -9055,14 +8814,14 @@ def assemble(self, user_dict, interview_status=None, old_user_dict=None, force_q break except UndefinedError as the_exception: # logmessage("UndefinedError") - if self.debug and docassemble.base.functions.this_thread.evaluation_context == 'docx': + if self.debug and this_thread.evaluation_context == 'docx': # logmessage(the_exception.__class__.__name__ + " exception during document assembly: " + str(the_exception) + "\n" + traceback.format_exc()) logmessage(the_exception.__class__.__name__ + " exception during document assembly: " + str(the_exception)) - docassemble.base.functions.reset_context() - missingVariable = extract_missing_name(the_exception) - # logmessage("extracted " + missingVariable) - variables_sought.add(missingVariable) - question_result = self.askfor(missingVariable, user_dict, old_user_dict, interview_status, seeking=interview_status.seeking, follow_mc=True) + reset_context() + missing_variable = extract_missing_name(the_exception) + # logmessage("extracted " + missing_variable) + variables_sought.add(missing_variable) + question_result = self.askfor(missing_variable, user_dict, old_user_dict, interview_status, seeking=interview_status.seeking, follow_mc=True) if question_result['type'] in ('continue', 're_run'): continue if question_result['type'] == 'refresh': @@ -9070,128 +8829,128 @@ def assemble(self, user_dict, interview_status=None, old_user_dict=None, force_q else: interview_status.populate(question_result) break - except CommandError as qError: + except CommandError as question_error: # logmessage("CommandError") - docassemble.base.functions.reset_context() - question_data = {'command': qError.return_type, 'sleep': qError.sleep, 'question': qError.question_text, 'subquestion': qError.subquestion_text} + reset_context() + question_data = {'command': question_error.return_type, 'sleep': question_error.sleep, 'question': question_error.question_text, 'subquestion': question_error.subquestion_text} new_interview_source = InterviewSourceString(content='') - new_interview = new_interview_source.get_interview() + new_interview = Interview(source=new_interview_source) reproduce_basics(self, new_interview) new_question = Question(question_data, new_interview, source=new_interview_source, package=self.source.package) new_question.name = "Question_Temp" interview_status.populate(new_question.ask(user_dict, old_user_dict, 'None', [], None, None)) break - except ResponseError as qError: - docassemble.base.functions.reset_context() + except ResponseError as question_error: + reset_context() # logmessage("Trapped ResponseError") question_data = {'extras': {}} - if hasattr(qError, 'response') and qError.response is not None: - question_data['response'] = qError.response - elif hasattr(qError, 'binaryresponse') and qError.binaryresponse is not None: - question_data['binaryresponse'] = qError.binaryresponse - elif hasattr(qError, 'filename') and qError.filename is not None: - question_data['response filename'] = qError.filename - elif hasattr(qError, 'url') and qError.url is not None: - question_data['redirect url'] = qError.url - elif hasattr(qError, 'all_variables') and qError.all_variables: - if hasattr(qError, 'include_internal'): - question_data['include_internal'] = qError.include_internal + if hasattr(question_error, 'response') and question_error.response is not None: + question_data['response'] = question_error.response + elif hasattr(question_error, 'binaryresponse') and question_error.binaryresponse is not None: + question_data['binaryresponse'] = question_error.binaryresponse + elif hasattr(question_error, 'filename') and question_error.filename is not None: + question_data['response filename'] = question_error.filename + elif hasattr(question_error, 'url') and question_error.url is not None: + question_data['redirect url'] = question_error.url + elif hasattr(question_error, 'all_variables') and question_error.all_variables: + if hasattr(question_error, 'include_internal'): + question_data['include_internal'] = question_error.include_internal question_data['content type'] = 'application/json' question_data['all_variables'] = True - elif hasattr(qError, 'nullresponse') and qError.nullresponse: - question_data['null response'] = qError.nullresponse - elif hasattr(qError, 'sleep') and qError.sleep: - question_data['sleep'] = qError.sleep - if hasattr(qError, 'content_type') and qError.content_type: - question_data['content type'] = qError.content_type - if hasattr(qError, 'response_code') and qError.response_code: - question_data['response code'] = qError.response_code + elif hasattr(question_error, 'nullresponse') and question_error.nullresponse: + question_data['null response'] = question_error.nullresponse + elif hasattr(question_error, 'sleep') and question_error.sleep: + question_data['sleep'] = question_error.sleep + if hasattr(question_error, 'content_type') and question_error.content_type: + question_data['content type'] = question_error.content_type + if hasattr(question_error, 'response_code') and question_error.response_code: + question_data['response code'] = question_error.response_code # new_interview = copy.deepcopy(self) # if self.source is None: # new_interview_source = InterviewSourceString(content='') # else: # new_interview_source = self.source new_interview_source = InterviewSourceString(content='') - new_interview = new_interview_source.get_interview() + new_interview = Interview(source=new_interview_source) reproduce_basics(self, new_interview) new_question = Question(question_data, new_interview, source=new_interview_source, package=self.source.package) new_question.name = "Question_Temp" # the_question = new_question.follow_multiple_choice(user_dict) interview_status.populate(new_question.ask(user_dict, old_user_dict, 'None', [], None, None)) break - except BackgroundResponseError as qError: - docassemble.base.functions.reset_context() + except BackgroundResponseError as question_error: + reset_context() # logmessage("Trapped BackgroundResponseError") question_data = {'extras': {}} - if hasattr(qError, 'backgroundresponse'): - question_data['backgroundresponse'] = normalize_background_response(qError.backgroundresponse) - if hasattr(qError, 'sleep'): - question_data['sleep'] = qError.sleep + if hasattr(question_error, 'backgroundresponse'): + question_data['backgroundresponse'] = normalize_background_response(question_error.backgroundresponse) + if hasattr(question_error, 'sleep'): + question_data['sleep'] = question_error.sleep new_interview_source = InterviewSourceString(content='') - new_interview = new_interview_source.get_interview() + new_interview = Interview(source=new_interview_source) reproduce_basics(self, new_interview) new_question = Question(question_data, new_interview, source=new_interview_source, package=self.source.package) new_question.name = "Question_Temp" interview_status.populate(new_question.ask(user_dict, old_user_dict, 'None', [], None, None)) break - except BackgroundResponseActionError as qError: - docassemble.base.functions.reset_context() + except BackgroundResponseActionError as question_error: + reset_context() # logmessage("Trapped BackgroundResponseActionError") question_data = {'extras': {}} - if hasattr(qError, 'action'): - question_data['action'] = qError.action + if hasattr(question_error, 'action'): + question_data['action'] = question_error.action new_interview_source = InterviewSourceString(content='') - new_interview = new_interview_source.get_interview() + new_interview = Interview(source=new_interview_source) reproduce_basics(self, new_interview) new_question = Question(question_data, new_interview, source=new_interview_source, package=self.source.package) new_question.name = "Question_Temp" interview_status.populate(new_question.ask(user_dict, old_user_dict, 'None', [], None, None)) break - # except SendFileError as qError: + # except SendFileError as question_error: # # logmessage("Trapped SendFileError") # question_data = {'extras': {}} - # if hasattr(qError, 'filename') and qError.filename is not None: - # question_data['response filename'] = qError.filename - # if hasattr(qError, 'content_type') and qError.content_type: - # question_data['content type'] = qError.content_type + # if hasattr(question_error, 'filename') and question_error.filename is not None: + # question_data['response filename'] = question_error.filename + # if hasattr(question_error, 'content_type') and question_error.content_type: + # question_data['content type'] = question_error.content_type # new_interview_source = InterviewSourceString(content='') - # new_interview = new_interview_source.get_interview() + # new_interview = Interview(source=new_interview_source) # new_question = Question(question_data, new_interview, source=new_interview_source, package=self.source.package) # new_question.name = "Question_Temp" # interview_status.populate(new_question.ask(user_dict, old_user_dict, 'None', [], None)) # break - except QuestionError as qError: + except QuestionError as question_error: # logmessage("QuestionError") - docassemble.base.functions.reset_context() + reset_context() question_data = {} - if qError.question: - question_data['question'] = qError.question - if qError.subquestion: - question_data['subquestion'] = qError.subquestion - if qError.reload: - question_data['reload'] = qError.reload - if qError.dead_end: + if question_error.question: + question_data['question'] = question_error.question + if question_error.subquestion: + question_data['subquestion'] = question_error.subquestion + if question_error.reload: + question_data['reload'] = question_error.reload + if question_error.dead_end: pass - elif qError.buttons: - question_data['buttons'] = qError.buttons + elif question_error.buttons: + question_data['buttons'] = question_error.buttons else: buttons = [] - if qError.show_exit is not False and not (qError.show_leave is True and qError.show_exit is None): + if question_error.show_exit is not False and not (question_error.show_leave is True and question_error.show_exit is None): exit_button = {word('Exit'): 'exit'} - if qError.url: - exit_button.update({'url': qError.url}) + if question_error.url: + exit_button.update({'url': question_error.url}) buttons.append(exit_button) - if qError.show_leave: + if question_error.show_leave: leave_button = {word('Leave'): 'leave'} - if qError.url: - leave_button.update({'url': qError.url}) + if question_error.url: + leave_button.update({'url': question_error.url}) buttons.append(leave_button) - if qError.show_restart is not False: + if question_error.show_restart is not False: buttons.append({word('Restart'): 'restart'}) if len(buttons) > 0: question_data['buttons'] = buttons new_interview_source = InterviewSourceString(content='') - new_interview = new_interview_source.get_interview() + new_interview = Interview(source=new_interview_source) reproduce_basics(self, new_interview) new_question = Question(question_data, new_interview, source=new_interview_source, package=self.source.package) new_question.name = "Question_Temp" @@ -9201,57 +8960,57 @@ def assemble(self, user_dict, interview_status=None, old_user_dict=None, force_q interview_status.populate(the_question.ask(user_dict, old_user_dict, 'None', [], None, None)) break except AttributeError as the_error: - if 'pending_error' in docassemble.base.functions.this_thread.misc: - del docassemble.base.functions.this_thread.misc['pending_error'] + if 'pending_error' in this_thread.misc: + del this_thread.misc['pending_error'] # logmessage("Regular attributeerror") - docassemble.base.functions.reset_context() + reset_context() # logmessage(str(the_error.args)) - docassemble.base.functions.wrap_up() + wrap_up() raise DASourceError('Got error ' + str(the_error) + " " + traceback.format_exc() + "\nHistory was " + pprint.pformat(interview_status.seeking)) except MandatoryQuestion: # logmessage("MandatoryQuestion") - docassemble.base.functions.reset_context() + reset_context() break except CodeExecute as code_error: # logmessage("CodeExecute") - docassemble.base.functions.reset_context() + reset_context() # if self.debug: # interview_status.seeking.append({'question': question, 'reason': 'mandatory code'}) exec(code_error.compute, user_dict) code_error.question.mark_as_answered(user_dict) - except SyntaxException as qError: + except SyntaxException as question_error: # logmessage("SyntaxException") - docassemble.base.functions.reset_context() + reset_context() the_question = None try: the_question = question except: pass - docassemble.base.functions.wrap_up() + wrap_up() if the_question is not None: - raise DASourceError(str(qError) + "\n\n" + str(self.idebug(self.data_for_debug))) - raise DASourceError("no question available: " + str(qError)) - except CompileException as qError: + raise DASourceError(str(question_error) + "\n\n" + str(self.idebug(self.data_for_debug))) + raise DASourceError("no question available: " + str(question_error)) + except CompileException as question_error: # logmessage("CompileException") - docassemble.base.functions.reset_context() + reset_context() the_question = None try: the_question = question except: pass - docassemble.base.functions.wrap_up() + wrap_up() if the_question is not None: - raise DASourceError(str(qError) + "\n\n" + str(self.idebug(self.data_for_debug))) - raise DASourceError("no question available: " + str(qError)) + raise DASourceError(str(question_error) + "\n\n" + str(self.idebug(self.data_for_debug))) + raise DASourceError("no question available: " + str(question_error)) else: - docassemble.base.functions.wrap_up() + wrap_up() raise DAErrorNoEndpoint('Docassemble has finished executing all code blocks marked as initial or mandatory, and finished asking all questions marked as mandatory (if any). It is a best practice to end your interview with a question that says goodbye.') except BaseException as the_error: # logmessage("Untrapped exception") if self.debug: the_error.interview = self the_error.interview_status = interview_status - the_error.user_dict = docassemble.base.functions.serializable_dict(user_dict) + the_error.user_dict = serializable_dict(user_dict) if not hasattr(the_error, '__traceback__'): cl, exc, tb = sys.exc_info() the_error.__traceback__ = tb @@ -9259,9 +9018,9 @@ def assemble(self, user_dict, interview_status=None, old_user_dict=None, force_q del exc del tb raise the_error - if docassemble.base.functions.this_thread.prevent_going_back: + if this_thread.prevent_going_back: interview_status.can_go_back = False - docassemble.base.functions.wrap_up() + wrap_up() if self.debug: interview_status.seeking.append({'done': True, 'time': time.time()}) @@ -9270,7 +9029,7 @@ def load_util(self, the_user_dict): if not self.consolidated_metadata.get('suppress loading util', False): exec(import_util, the_user_dict) - def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, **kwargs): + def askfor(self, missing_variable, user_dict, old_user_dict, interview_status, **kwargs): seeking_question = kwargs.get('seeking_question', False) variable_stack = kwargs.get('variable_stack', set()) questions_tried = kwargs.get('questions_tried', {}) @@ -9281,35 +9040,35 @@ def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, ** follow_mc = kwargs.get('follow_mc', True) seeking = kwargs.get('seeking', []) if self.debug: - seeking.append({'variable': missingVariable, 'time': time.time()}) + seeking.append({'variable': missing_variable, 'time': time.time()}) if recursion_depth > self.recursion_limit: raise DASourceError("There appears to be an infinite loop. Variables in stack are " + ", ".join(sorted(variable_stack)) + ".") - # logmessage("askfor: I don't have " + str(missingVariable) + " for language " + str(language)) - # logmessage("I don't have " + str(missingVariable) + " for language " + str(language)) - origMissingVariable = missingVariable - docassemble.base.functions.set_current_variable(origMissingVariable) - # if missingVariable in variable_stack: - # raise DASourceError("Infinite loop: " + missingVariable + " already looked for, where stack is " + str(variable_stack)) - # variable_stack.add(missingVariable) + # logmessage("askfor: I don't have " + str(missing_variable) + " for language " + str(language)) + # logmessage("I don't have " + str(missing_variable) + " for language " + str(language)) + orig_missing_variable = missing_variable + set_current_variable(orig_missing_variable) + # if missing_variable in variable_stack: + # raise DASourceError("Infinite loop: " + missing_variable + " already looked for, where stack is " + str(variable_stack)) + # variable_stack.add(missing_variable) # found_generic = False - # realMissingVariable = missingVariable + # realMissingVariable = missing_variable totry = [] variants = [] level_dict = {} generic_dict = {} - expression_as_list = [x for x in match_brackets_or_dot.split(missingVariable) if x != ''] + expression_as_list = [x for x in match_brackets_or_dot.split(missing_variable) if x != ''] expression_as_list.append('') recurse_indices(expression_as_list, list_of_indices, [], variants, level_dict, [], generic_dict, []) # logmessage("variants: " + repr(variants)) for variant in variants: - totry.append({'real': missingVariable, 'vari': variant, 'iterators': level_dict[variant], 'generic': generic_dict[variant], 'is_generic': 0 if generic_dict[variant] == '' else 1, 'num_dots': variant.count('.'), 'num_iterators': variant.count('[')}) + totry.append({'real': missing_variable, 'vari': variant, 'iterators': level_dict[variant], 'generic': generic_dict[variant], 'is_generic': 0 if generic_dict[variant] == '' else 1, 'num_dots': variant.count('.'), 'num_iterators': variant.count('[')}) totry = sorted(sorted(sorted(sorted(totry, key=lambda x: len(x['iterators'])), key=lambda x: x['num_iterators'], reverse=True), key=lambda x: x['num_dots'], reverse=True), key=lambda x: x['is_generic']) # logmessage("ask_for: totry is " + "\n".join([x['vari'] for x in totry])) questions_to_try = [] for mv in totry: # realMissingVariable = mv['real'] - missingVariable = mv['vari'] - # logmessage("Trying missingVariable " + missingVariable) + missing_variable = mv['vari'] + # logmessage("Trying missing_variable " + missing_variable) if mv['is_generic']: # logmessage("Testing out generic " + mv['generic']) try: @@ -9318,22 +9077,22 @@ def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, ** classes_to_look_for = [type(root_evaluated).__name__] recursive_add_classes(classes_to_look_for, type(root_evaluated)) for generic_object in classes_to_look_for: - # logmessage("Looking for generic object " + generic_object + " for " + missingVariable) - if generic_object in self.generic_questions and missingVariable in self.generic_questions[generic_object] and (language in self.generic_questions[generic_object][missingVariable] or '*' in self.generic_questions[generic_object][missingVariable]): + # logmessage("Looking for generic object " + generic_object + " for " + missing_variable) + if generic_object in self.generic_questions and missing_variable in self.generic_questions[generic_object] and (language in self.generic_questions[generic_object][missing_variable] or '*' in self.generic_questions[generic_object][missing_variable]): for lang in [language, '*']: - if lang in self.generic_questions[generic_object][missingVariable]: - for the_question_to_use in self.sort_with_orderings(self.generic_questions[generic_object][missingVariable][lang]): - questions_to_try.append((the_question_to_use, True, mv['generic'], mv['iterators'], missingVariable, generic_object)) + if lang in self.generic_questions[generic_object][missing_variable]: + for the_question_to_use in self.sort_with_orderings(self.generic_questions[generic_object][missing_variable][lang]): + questions_to_try.append((the_question_to_use, True, mv['generic'], mv['iterators'], missing_variable, generic_object)) except: pass continue # logmessage("askfor: questions to try is " + str(questions_to_try)) - if missingVariable in self.questions: + if missing_variable in self.questions: for lang in [language, '*']: # logmessage("lang is " + lang) - if lang in self.questions[missingVariable]: - for the_question in self.sort_with_orderings(self.questions[missingVariable][lang]): - questions_to_try.append((the_question, False, 'None', mv['iterators'], missingVariable, None)) + if lang in self.questions[missing_variable]: + for the_question in self.sort_with_orderings(self.questions[missing_variable][lang]): + questions_to_try.append((the_question, False, 'None', mv['iterators'], missing_variable, None)) # logmessage("askfor: questions to try is " + str(questions_to_try)) num_cycles = 0 missing_var = "_unknown" @@ -9342,7 +9101,7 @@ def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, ** if num_cycles > self.loop_limit: raise DASourceError("Infinite loop detected while looking for " + missing_var) a_question_was_skipped = False - docassemble.base.functions.reset_gathering_mode(origMissingVariable) + reset_gathering_mode(orig_missing_variable) # logmessage("Starting the while loop") try: for the_question, is_generic, the_x, iterators, missing_var, generic_object in questions_to_try: @@ -9399,7 +9158,7 @@ def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, ** skip_question = None for field in question.fields: if hasattr(field, 'showif_code') and hasattr(field, 'saveas') and field.saveas == field_id: - docassemble.base.functions.this_thread.misc['current_field'] = field.number + this_thread.misc['current_field'] = field.number result = eval(field.showif_code, user_dict) if hasattr(field, 'extras') and 'show_if_sign_code' in field.extras and field.extras['show_if_sign_code'] == 0: if result: @@ -9429,18 +9188,18 @@ def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, ** gathered = eval(question.gathered, user_dict) thename = substitute_vars(from_safeid(question.fields[0].saveas), is_generic, the_x, iterators) if question.use_objects == 'objects': - user_dict['_DADATA'] = docassemble.base.util.objects_from_data(recursive_eval_dataobject(question.fields[0].data, user_dict), recursive=True, gathered=gathered, name=thename, package=question.package) + user_dict['_DADATA'] = objects_from_data(recursive_eval_dataobject(question.fields[0].data, user_dict), recursive=True, gathered=gathered, name=thename, package=question.package) elif question.use_objects: - user_dict['_DADATA'] = docassemble.base.util.objects_from_structure(recursive_eval_dataobject(question.fields[0].data, user_dict), root=thename, gathered=gathered) + user_dict['_DADATA'] = objects_from_structure(recursive_eval_dataobject(question.fields[0].data, user_dict), root=thename, gathered=gathered) else: user_dict['_DADATA'] = recursive_eval_dataobject(question.fields[0].data, user_dict) the_string = thename + ' = _DADATA' exec(the_string, user_dict) del user_dict['_DADATA'] question.post_exec(user_dict) - docassemble.base.functions.pop_current_variable() + pop_current_variable() question.invalidate_dependencies(user_dict, old_values) - return {'type': 'continue', 'sought': missing_var, 'orig_sought': origMissingVariable} + return {'type': 'continue', 'sought': missing_var, 'orig_sought': orig_missing_variable} if question.question_type == "data_from_code": question.exec_setup(is_generic, the_x, iterators, user_dict) old_values = question.get_old_values(user_dict) @@ -9450,22 +9209,22 @@ def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, ** gathered = eval(question.gathered, user_dict) thename = substitute_vars(from_safeid(question.fields[0].saveas), is_generic, the_x, iterators) if question.use_objects == 'objects': - user_dict['_DADATAFROMCODE'] = docassemble.base.util.objects_from_data(recursive_eval_data_from_code(question.fields[0].data, user_dict), recursive=True, gathered=gathered, name=thename, package=question.package) + user_dict['_DADATAFROMCODE'] = objects_from_data(recursive_eval_data_from_code(question.fields[0].data, user_dict), recursive=True, gathered=gathered, name=thename, package=question.package) elif question.use_objects: - user_dict['_DADATAFROMCODE'] = docassemble.base.util.objects_from_structure(recursive_eval_data_from_code(question.fields[0].data, user_dict), root=thename, gathered=gathered) + user_dict['_DADATAFROMCODE'] = objects_from_structure(recursive_eval_data_from_code(question.fields[0].data, user_dict), root=thename, gathered=gathered) else: user_dict['_DADATAFROMCODE'] = recursive_eval_data_from_code(question.fields[0].data, user_dict) the_string = thename + ' = _DADATAFROMCODE' exec(the_string, user_dict) del user_dict['_DADATAFROMCODE'] question.post_exec(user_dict) - docassemble.base.functions.pop_current_variable() + pop_current_variable() question.invalidate_dependencies(user_dict, old_values) - return {'type': 'continue', 'sought': missing_var, 'orig_sought': origMissingVariable} + return {'type': 'continue', 'sought': missing_var, 'orig_sought': orig_missing_variable} if question.question_type == "objects_from_file": question.exec_setup(is_generic, the_x, iterators, user_dict) old_variable = None - docassemble.base.functions.this_thread.current_question = question + this_thread.current_question = question exec(import_core, user_dict) if isinstance(question.use_objects, (bool, NoneType)): use_objects = bool(question.use_objects) @@ -9500,15 +9259,15 @@ def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, ** pass continue question.post_exec(user_dict) - docassemble.base.functions.pop_current_variable() + pop_current_variable() if old_variable is not None: question.invalidate_dependencies_of_variable(user_dict, missing_var, old_variable) - return {'type': 'continue', 'sought': missing_var, 'orig_sought': origMissingVariable} + return {'type': 'continue', 'sought': missing_var, 'orig_sought': orig_missing_variable} if question.question_type == "objects": question.exec_setup(is_generic, the_x, iterators, user_dict) success = False old_variable = None - docassemble.base.functions.this_thread.current_question = question + this_thread.current_question = question for keyvalue in question.objects: # logmessage("In a for loop for keyvalue") for raw_variable, object_type_name in keyvalue.items(): @@ -9568,11 +9327,11 @@ def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, ** # question.mark_as_answered(user_dict) # logmessage("pop current variable") question.post_exec(user_dict) - docassemble.base.functions.pop_current_variable() + pop_current_variable() if old_variable is not None: question.invalidate_dependencies_of_variable(user_dict, missing_var, old_variable) # logmessage("Returning") - return {'type': 'continue', 'sought': missing_var, 'orig_sought': origMissingVariable} + return {'type': 'continue', 'sought': missing_var, 'orig_sought': orig_missing_variable} if question.question_type == "template": question.exec_setup(is_generic, the_x, iterators, user_dict) temp_vars = {} @@ -9583,13 +9342,13 @@ def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, ** for indexno in range(len(iterators)): temp_vars[list_of_indices[indexno]] = user_dict[list_of_indices[indexno]] if question.target is not None: - return {'type': 'template', 'question_text': question.content.text(user_dict).rstrip(), 'subquestion_text': None, 'continue_label': None, 'audiovideo': None, 'decorations': None, 'help_text': None, 'interview_help_text': None, 'attachments': None, 'question': question, 'selectcompute': {}, 'defaults': {}, 'hints': {}, 'helptexts': {}, 'extras': {}, 'labels': {}, 'sought': missing_var, 'orig_sought': origMissingVariable} + return {'type': 'template', 'question_text': question.content.text(user_dict).rstrip(), 'subquestion_text': None, 'continue_label': None, 'audiovideo': None, 'decorations': None, 'help_text': None, 'interview_help_text': None, 'attachments': None, 'question': question, 'selectcompute': {}, 'defaults': {}, 'hints': {}, 'helptexts': {}, 'extras': {}, 'labels': {}, 'sought': missing_var, 'orig_sought': orig_missing_variable} if question.decorations is None: decoration_list = [] else: decoration_list = question.decorations actual_saveas = substitute_vars(from_safeid(question.fields[0].saveas), is_generic, the_x, iterators) - # docassemble.base.functions.this_thread.template_vars.append(actual_saveas) + # this_thread.template_vars.append(actual_saveas) found_object = False try: the_object = eval(actual_saveas, user_dict) @@ -9611,8 +9370,8 @@ def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, ** the_object.userdict = user_dict the_object.tempvars = temp_vars question.post_exec(user_dict) - docassemble.base.functions.pop_current_variable() - return {'type': 'continue', 'sought': missing_var, 'orig_sought': origMissingVariable} + pop_current_variable() + return {'type': 'continue', 'sought': missing_var, 'orig_sought': orig_missing_variable} if question.question_type == "template_code": question.exec_setup(is_generic, the_x, iterators, user_dict) the_filenames = eval(question.compute, user_dict) @@ -9635,7 +9394,7 @@ def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, ** raise DASourceError("askfor: error downloading " + str(the_filename) + ": " + str(err)) the_filename = temp_template_file.name else: - the_filename = docassemble.base.functions.package_template_filename(the_filename, package=question.package) + the_filename = package_template_filename(the_filename, package=question.package) else: the_filename = None if the_filename is None or not os.path.isfile(the_filename): @@ -9675,8 +9434,8 @@ def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, ** the_object.userdict = user_dict the_object.tempvars = temp_vars question.post_exec(user_dict) - docassemble.base.functions.pop_current_variable() - return {'type': 'continue', 'sought': missing_var, 'orig_sought': origMissingVariable} + pop_current_variable() + return {'type': 'continue', 'sought': missing_var, 'orig_sought': orig_missing_variable} if question.question_type == "table": question.exec_setup(is_generic, the_x, iterators, user_dict) temp_vars = {} @@ -9702,7 +9461,7 @@ def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, ** table_info.filter_expression = question.fields[0].extras['filter_expression'] table_info.saveas = from_safeid(question.fields[0].saveas) actual_saveas = substitute_vars(table_info.saveas, is_generic, the_x, iterators) - # docassemble.base.functions.this_thread.template_vars.append(actual_saveas) + # this_thread.template_vars.append(actual_saveas) the_string = "from docassemble.base.util import DALazyTableTemplate" exec(the_string, user_dict) found_object = False @@ -9723,13 +9482,13 @@ def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, ** the_object.tempvars = temp_vars # logmessage("Pop variable for table") question.post_exec(user_dict) - docassemble.base.functions.pop_current_variable() - return {'type': 'continue', 'sought': missing_var, 'orig_sought': origMissingVariable} + pop_current_variable() + return {'type': 'continue', 'sought': missing_var, 'orig_sought': orig_missing_variable} if question.question_type == 'attachments': question.exec_setup(is_generic, the_x, iterators, user_dict) old_values = question.get_old_values(user_dict) - # logmessage("original missing variable is " + origMissingVariable) - question.processed_attachments(user_dict, seeking_var=origMissingVariable, use_cache=False) + # logmessage("original missing variable is " + orig_missing_variable) + question.processed_attachments(user_dict, seeking_var=orig_missing_variable, use_cache=False) if missing_var in variable_stack: variable_stack.remove(missing_var) try: @@ -9739,9 +9498,9 @@ def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, ** logmessage("Problem with attachments block: " + err.__class__.__name__ + ": " + str(err)) continue question.post_exec(user_dict) - docassemble.base.functions.pop_current_variable() + pop_current_variable() question.invalidate_dependencies(user_dict, old_values) - return {'type': 'continue', 'sought': missing_var, 'orig_sought': origMissingVariable} + return {'type': 'continue', 'sought': missing_var, 'orig_sought': orig_missing_variable} if question.question_type in ["code", "event_code"]: question.exec_setup(is_generic, the_x, iterators, user_dict) was_defined = False @@ -9753,8 +9512,8 @@ def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, ** except: pass if question.question_type == 'event_code': - docassemble.base.functions.pop_event_stack(origMissingVariable) - docassemble.base.functions.this_thread.current_question = question + pop_event_stack(orig_missing_variable) + this_thread.current_question = question if was_defined: exec_with_trap(question, user_dict, old_variable=missing_var) else: @@ -9763,12 +9522,12 @@ def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, ** if missing_var in variable_stack: variable_stack.remove(missing_var) if question.question_type == 'event_code': - docassemble.base.functions.pop_current_variable() - docassemble.base.functions.pop_event_stack(origMissingVariable) + pop_current_variable() + pop_event_stack(orig_missing_variable) question.invalidate_dependencies(user_dict, old_values) if was_defined: exec("del __oldvariable__", user_dict) - return {'type': 'continue', 'sought': missing_var, 'orig_sought': origMissingVariable} + return {'type': 'continue', 'sought': missing_var, 'orig_sought': orig_missing_variable} try: eval(missing_var, user_dict) if was_defined: @@ -9777,9 +9536,9 @@ def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, ** if seeking_question: continue # question.mark_as_answered(user_dict) - docassemble.base.functions.pop_current_variable() - docassemble.base.functions.pop_event_stack(origMissingVariable) - return {'type': 'continue', 'sought': missing_var, 'orig_sought': origMissingVariable} + pop_current_variable() + pop_event_stack(orig_missing_variable) + return {'type': 'continue', 'sought': missing_var, 'orig_sought': orig_missing_variable} except: if was_defined: try: @@ -9793,35 +9552,35 @@ def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, ** interview_status.mark_tentative_as_answered(user_dict) if question.question_type == 'continue': continue - return question.ask(user_dict, old_user_dict, the_x, iterators, missing_var, origMissingVariable) + return question.ask(user_dict, old_user_dict, the_x, iterators, missing_var, orig_missing_variable) if a_question_was_skipped: - raise DASourceError("Infinite loop: " + missingVariable + " already looked for, where stack is " + str(variable_stack)) - if 'forgive_missing_question' in docassemble.base.functions.this_thread.misc and origMissingVariable in docassemble.base.functions.this_thread.misc['forgive_missing_question']: - docassemble.base.functions.pop_current_variable() - docassemble.base.functions.pop_event_stack(origMissingVariable) - if 'action' in docassemble.base.functions.this_thread.current_info and docassemble.base.functions.this_thread.current_info['action'] == origMissingVariable: - del docassemble.base.functions.this_thread.current_info['action'] - return {'type': 'continue', 'sought': origMissingVariable, 'orig_sought': origMissingVariable} - if self.options.get('use catchall', False) and not origMissingVariable.endswith('.value'): + raise DASourceError("Infinite loop: " + missing_variable + " already looked for, where stack is " + str(variable_stack)) + if 'forgive_missing_question' in this_thread.misc and orig_missing_variable in this_thread.misc['forgive_missing_question']: + pop_current_variable() + pop_event_stack(orig_missing_variable) + if 'action' in this_thread.current_info and this_thread.current_info['action'] == orig_missing_variable: + del this_thread.current_info['action'] + return {'type': 'continue', 'sought': orig_missing_variable, 'orig_sought': orig_missing_variable} + if self.options.get('use catchall', False) and not orig_missing_variable.endswith('.value'): the_string = "from docassemble.base.util import DACatchAll" exec(the_string, user_dict) - the_string = origMissingVariable + ' = DACatchAll(' + repr(origMissingVariable) + ')' + the_string = orig_missing_variable + ' = DACatchAll(' + repr(orig_missing_variable) + ')' exec(the_string, user_dict) - docassemble.base.functions.pop_current_variable() - docassemble.base.functions.pop_event_stack(origMissingVariable) - return {'type': 'continue', 'sought': origMissingVariable, 'orig_sought': origMissingVariable} - raise DAErrorMissingVariable("Interview has an error. There was a reference to a variable '" + origMissingVariable + "' that could not be looked up in the question file (for language '" + str(language) + "') or in any of the files incorporated by reference into the question file.", variable=origMissingVariable) + pop_current_variable() + pop_event_stack(orig_missing_variable) + return {'type': 'continue', 'sought': orig_missing_variable, 'orig_sought': orig_missing_variable} + raise DAErrorMissingVariable("Interview has an error. There was a reference to a variable '" + orig_missing_variable + "' that could not be looked up in the question file (for language '" + str(language) + "') or in any of the files incorporated by reference into the question file.", variable=orig_missing_variable) except ForcedReRun: - docassemble.base.functions.pop_current_variable() - docassemble.base.functions.pop_event_stack(origMissingVariable) - return {'type': 're_run', 'sought': origMissingVariable, 'orig_sought': origMissingVariable} + pop_current_variable() + pop_event_stack(orig_missing_variable) + return {'type': 're_run', 'sought': orig_missing_variable, 'orig_sought': orig_missing_variable} except (NameError, UnboundLocalError, DAAttributeError, DAIndexError) as the_exception: - if 'pending_error' in docassemble.base.functions.this_thread.misc: - del docassemble.base.functions.this_thread.misc['pending_error'] + if 'pending_error' in this_thread.misc: + del this_thread.misc['pending_error'] # logmessage("Error in " + the_exception.__class__.__name__ + " is " + str(the_exception)) - if self.debug and docassemble.base.functions.this_thread.evaluation_context == 'docx': + if self.debug and this_thread.evaluation_context == 'docx': logmessage("NameError exception during document assembly: " + str(the_exception)) - docassemble.base.functions.reset_context() + reset_context() seeking_question = False if isinstance(the_exception, ForcedNameError): # logmessage("askfor: got a ForcedNameError for " + str(the_exception.name)) @@ -9829,7 +9588,7 @@ def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, ** seeking_question = True # logmessage("Seeking question is True") exception_name = the_exception.name - newMissingVariable = exception_name + new_missing_variable = exception_name if the_exception.next_action is not None and not interview_status.checkin: if 'event_stack' not in user_dict['_internal']: user_dict['_internal']['event_stack'] = {} @@ -9848,12 +9607,12 @@ def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, ** if len(new_items) > 0: user_dict['_internal']['event_stack'][session_uid] = new_items + user_dict['_internal']['event_stack'][session_uid] if the_exception.arguments is not None: - docassemble.base.functions.this_thread.current_info.update({'action': exception_name, 'arguments': the_exception.arguments}) + this_thread.current_info.update({'action': exception_name, 'arguments': the_exception.arguments}) if exception_name.startswith('_da_'): - docassemble.base.functions.pop_current_variable() - docassemble.base.functions.pop_event_stack(origMissingVariable) - return {'type': 're_run', 'sought': origMissingVariable, 'orig_sought': origMissingVariable} - docassemble.base.functions.this_thread.misc['forgive_missing_question'] = [exception_name] + pop_current_variable() + pop_event_stack(orig_missing_variable) + return {'type': 're_run', 'sought': orig_missing_variable, 'orig_sought': orig_missing_variable} + this_thread.misc['forgive_missing_question'] = [exception_name] else: # logmessage("regular nameerror") if type(the_exception) is NameError: # pylint: disable=unidiomatic-typecheck @@ -9871,171 +9630,171 @@ def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, ** del exc del tb follow_mc = True - newMissingVariable = extract_missing_name(the_exception) - if newMissingVariable == 'file': + new_missing_variable = extract_missing_name(the_exception) + if new_missing_variable == 'file': raise - # newMissingVariable = str(the_exception).split("'")[1] - # if newMissingVariable in questions_tried and newMissingVariable in variable_stack: - # raise DASourceError("Infinite loop: " + missingVariable + " already looked for, where stack is " + str(variable_stack)) - if newMissingVariable not in questions_tried: - questions_tried[newMissingVariable] = set() + # new_missing_variable = str(the_exception).split("'")[1] + # if new_missing_variable in questions_tried and new_missing_variable in variable_stack: + # raise DASourceError("Infinite loop: " + missing_variable + " already looked for, where stack is " + str(variable_stack)) + if new_missing_variable not in questions_tried: + questions_tried[new_missing_variable] = set() else: - variable_stack.add(missingVariable) + variable_stack.add(missing_variable) if current_question.question_type != 'objects': - questions_tried[newMissingVariable].add(current_question) + questions_tried[new_missing_variable].add(current_question) try: - eval(origMissingVariable, user_dict) + eval(orig_missing_variable, user_dict) was_defined = True except: was_defined = False - question_result = self.askfor(newMissingVariable, user_dict, old_user_dict, interview_status, variable_stack=variable_stack, questions_tried=questions_tried, seeking=seeking, follow_mc=follow_mc, recursion_depth=recursion_depth, seeking_question=seeking_question) - if question_result['type'] == 'continue' and missing_var != newMissingVariable: + question_result = self.askfor(new_missing_variable, user_dict, old_user_dict, interview_status, variable_stack=variable_stack, questions_tried=questions_tried, seeking=seeking, follow_mc=follow_mc, recursion_depth=recursion_depth, seeking_question=seeking_question) + if question_result['type'] == 'continue' and missing_var != new_missing_variable: if not was_defined: try: - eval(origMissingVariable, user_dict) + eval(orig_missing_variable, user_dict) now_defined = True except: now_defined = False if now_defined: - docassemble.base.functions.pop_current_variable() - return {'type': 'continue', 'sought': missing_var, 'orig_sought': origMissingVariable} - # logmessage("Continuing after asking for newMissingVariable " + str(newMissingVariable)) + pop_current_variable() + return {'type': 'continue', 'sought': missing_var, 'orig_sought': orig_missing_variable} + # logmessage("Continuing after asking for new_missing_variable " + str(new_missing_variable)) continue - docassemble.base.functions.pop_current_variable() + pop_current_variable() return question_result except UndefinedError as the_exception: # logmessage("UndefinedError") - if self.debug and docassemble.base.functions.this_thread.evaluation_context == 'docx': + if self.debug and this_thread.evaluation_context == 'docx': # logmessage(the_exception.__class__.__name__ + " exception during document assembly: " + str(the_exception) + "\n" + traceback.format_exc()) logmessage(the_exception.__class__.__name__ + " exception during document assembly: " + str(the_exception)) - docassemble.base.functions.reset_context() - newMissingVariable = extract_missing_name(the_exception) - if newMissingVariable not in questions_tried: - questions_tried[newMissingVariable] = set() + reset_context() + new_missing_variable = extract_missing_name(the_exception) + if new_missing_variable not in questions_tried: + questions_tried[new_missing_variable] = set() else: - variable_stack.add(missingVariable) + variable_stack.add(missing_variable) if current_question.question_type != 'objects': - questions_tried[newMissingVariable].add(current_question) - question_result = self.askfor(newMissingVariable, user_dict, old_user_dict, interview_status, variable_stack=variable_stack, questions_tried=questions_tried, seeking=seeking, follow_mc=True, recursion_depth=recursion_depth, seeking_question=seeking_question) + questions_tried[new_missing_variable].add(current_question) + question_result = self.askfor(new_missing_variable, user_dict, old_user_dict, interview_status, variable_stack=variable_stack, questions_tried=questions_tried, seeking=seeking, follow_mc=True, recursion_depth=recursion_depth, seeking_question=seeking_question) if question_result['type'] == 'continue': continue - docassemble.base.functions.pop_current_variable() + pop_current_variable() return question_result - except CommandError as qError: - # logmessage("CommandError: " + str(qError)) - docassemble.base.functions.reset_context() - question_data = {'command': qError.return_type, 'sleep': qError.sleep, 'question': qError.question_text, 'subquestion': qError.subquestion_text} + except CommandError as question_error: + # logmessage("CommandError: " + str(question_error)) + reset_context() + question_data = {'command': question_error.return_type, 'sleep': question_error.sleep, 'question': question_error.question_text, 'subquestion': question_error.subquestion_text} new_interview_source = InterviewSourceString(content='') - new_interview = new_interview_source.get_interview() + new_interview = Interview(source=new_interview_source) reproduce_basics(self, new_interview) new_question = Question(question_data, new_interview, source=new_interview_source, package=self.source.package) new_question.name = "Question_Temp" - return new_question.ask(user_dict, old_user_dict, 'None', [], missing_var, origMissingVariable) - except ResponseError as qError: + return new_question.ask(user_dict, old_user_dict, 'None', [], missing_var, orig_missing_variable) + except ResponseError as question_error: # logmessage("ResponseError") - docassemble.base.functions.reset_context() + reset_context() # logmessage("Trapped ResponseError2") question_data = {'extras': {}} - if hasattr(qError, 'response') and qError.response is not None: - question_data['response'] = qError.response - elif hasattr(qError, 'binaryresponse') and qError.binaryresponse is not None: - question_data['binaryresponse'] = qError.binaryresponse - elif hasattr(qError, 'filename') and qError.filename is not None: - question_data['response filename'] = qError.filename - elif hasattr(qError, 'url') and qError.url is not None: - question_data['redirect url'] = qError.url - elif hasattr(qError, 'all_variables') and qError.all_variables: - if hasattr(qError, 'include_internal'): - question_data['include_internal'] = qError.include_internal + if hasattr(question_error, 'response') and question_error.response is not None: + question_data['response'] = question_error.response + elif hasattr(question_error, 'binaryresponse') and question_error.binaryresponse is not None: + question_data['binaryresponse'] = question_error.binaryresponse + elif hasattr(question_error, 'filename') and question_error.filename is not None: + question_data['response filename'] = question_error.filename + elif hasattr(question_error, 'url') and question_error.url is not None: + question_data['redirect url'] = question_error.url + elif hasattr(question_error, 'all_variables') and question_error.all_variables: + if hasattr(question_error, 'include_internal'): + question_data['include_internal'] = question_error.include_internal question_data['content type'] = 'application/json' question_data['all_variables'] = True - elif hasattr(qError, 'nullresponse') and qError.nullresponse: - question_data['null response'] = qError.nullresponse - elif hasattr(qError, 'sleep') and qError.sleep: - question_data['sleep'] = qError.sleep - if hasattr(qError, 'content_type') and qError.content_type: - question_data['content type'] = qError.content_type - if hasattr(qError, 'response_code') and qError.response_code: - question_data['response code'] = qError.response_code + elif hasattr(question_error, 'nullresponse') and question_error.nullresponse: + question_data['null response'] = question_error.nullresponse + elif hasattr(question_error, 'sleep') and question_error.sleep: + question_data['sleep'] = question_error.sleep + if hasattr(question_error, 'content_type') and question_error.content_type: + question_data['content type'] = question_error.content_type + if hasattr(question_error, 'response_code') and question_error.response_code: + question_data['response code'] = question_error.response_code new_interview_source = InterviewSourceString(content='') - new_interview = new_interview_source.get_interview() + new_interview = Interview(source=new_interview_source) reproduce_basics(self, new_interview) new_question = Question(question_data, new_interview, source=new_interview_source, package=self.source.package) new_question.name = "Question_Temp" # the_question = new_question.follow_multiple_choice(user_dict) - docassemble.base.functions.pop_event_stack(origMissingVariable) - return new_question.ask(user_dict, old_user_dict, 'None', [], missing_var, origMissingVariable) - except BackgroundResponseError as qError: + pop_event_stack(orig_missing_variable) + return new_question.ask(user_dict, old_user_dict, 'None', [], missing_var, orig_missing_variable) + except BackgroundResponseError as question_error: # logmessage("BackgroundResponseError") - docassemble.base.functions.reset_context() + reset_context() # logmessage("Trapped BackgroundResponseError2") question_data = {'extras': {}} - if hasattr(qError, 'backgroundresponse'): - question_data['backgroundresponse'] = normalize_background_response(qError.backgroundresponse) - if hasattr(qError, 'sleep'): - question_data['sleep'] = qError.sleep + if hasattr(question_error, 'backgroundresponse'): + question_data['backgroundresponse'] = normalize_background_response(question_error.backgroundresponse) + if hasattr(question_error, 'sleep'): + question_data['sleep'] = question_error.sleep new_interview_source = InterviewSourceString(content='') - new_interview = new_interview_source.get_interview() + new_interview = Interview(source=new_interview_source) reproduce_basics(self, new_interview) new_question = Question(question_data, new_interview, source=new_interview_source, package=self.source.package) new_question.name = "Question_Temp" - docassemble.base.functions.pop_event_stack(origMissingVariable) - return new_question.ask(user_dict, old_user_dict, 'None', [], missing_var, origMissingVariable) - except BackgroundResponseActionError as qError: + pop_event_stack(orig_missing_variable) + return new_question.ask(user_dict, old_user_dict, 'None', [], missing_var, orig_missing_variable) + except BackgroundResponseActionError as question_error: # logmessage("BackgroundResponseActionError") - docassemble.base.functions.reset_context() + reset_context() # logmessage("Trapped BackgroundResponseActionError2") question_data = {'extras': {}} - if hasattr(qError, 'action'): - question_data['action'] = qError.action + if hasattr(question_error, 'action'): + question_data['action'] = question_error.action new_interview_source = InterviewSourceString(content='') - new_interview = new_interview_source.get_interview() + new_interview = Interview(source=new_interview_source) reproduce_basics(self, new_interview) new_question = Question(question_data, new_interview, source=new_interview_source, package=self.source.package) new_question.name = "Question_Temp" - docassemble.base.functions.pop_event_stack(origMissingVariable) - return new_question.ask(user_dict, old_user_dict, 'None', [], missing_var, origMissingVariable) - except QuestionError as qError: + pop_event_stack(orig_missing_variable) + return new_question.ask(user_dict, old_user_dict, 'None', [], missing_var, orig_missing_variable) + except QuestionError as question_error: # logmessage("QuestionError") - docassemble.base.functions.reset_context() + reset_context() # logmessage("Trapped QuestionError") question_data = {} - if qError.question: - question_data['question'] = qError.question - if qError.subquestion: - question_data['subquestion'] = qError.subquestion - if qError.dead_end: + if question_error.question: + question_data['question'] = question_error.question + if question_error.subquestion: + question_data['subquestion'] = question_error.subquestion + if question_error.dead_end: pass - elif qError.buttons: - question_data['buttons'] = qError.buttons + elif question_error.buttons: + question_data['buttons'] = question_error.buttons else: buttons = [] - if qError.show_exit is not False and not (qError.show_leave is True and qError.show_exit is None): + if question_error.show_exit is not False and not (question_error.show_leave is True and question_error.show_exit is None): exit_button = {word('Exit'): 'exit'} - if qError.url: - exit_button.update({'url': qError.url}) + if question_error.url: + exit_button.update({'url': question_error.url}) buttons.append(exit_button) - if qError.show_leave: + if question_error.show_leave: leave_button = {word('Leave'): 'leave'} - if qError.url: - leave_button.update({'url': qError.url}) + if question_error.url: + leave_button.update({'url': question_error.url}) buttons.append(leave_button) - if qError.show_restart is not False: + if question_error.show_restart is not False: buttons.append({word('Restart'): 'restart'}) if len(buttons) > 0: question_data['buttons'] = buttons new_interview_source = InterviewSourceString(content='') - new_interview = new_interview_source.get_interview() + new_interview = Interview(source=new_interview_source) reproduce_basics(self, new_interview) new_question = Question(question_data, new_interview, source=new_interview_source, package=self.source.package) new_question.name = "Question_Temp" new_question.embeds = True # will this be a problem? yup the_question = new_question.follow_multiple_choice(user_dict, interview_status, False, 'None', []) - return the_question.ask(user_dict, old_user_dict, 'None', [], missing_var, origMissingVariable) + return the_question.ask(user_dict, old_user_dict, 'None', [], missing_var, orig_missing_variable) except CodeExecute as code_error: # logmessage("CodeExecute") - docassemble.base.functions.reset_context() + reset_context() # if self.debug: # interview_status.seeking.append({'question': question, 'reason': 'mandatory code'}) # logmessage("Going to execute " + str(code_error.compute) + " where missing_var is " + str(missing_var)) @@ -10045,51 +9804,51 @@ def askfor(self, missingVariable, user_dict, old_user_dict, interview_status, ** code_error.question.mark_as_answered(user_dict) # logmessage("Got here 1") # logmessage("returning from running code") - docassemble.base.functions.pop_current_variable() + pop_current_variable() # logmessage("Got here 2") - return {'type': 'continue', 'sought': missing_var, 'orig_sought': origMissingVariable} + return {'type': 'continue', 'sought': missing_var, 'orig_sought': orig_missing_variable} except: # raise DASourceError("Problem setting that variable") continue - except SyntaxException as qError: + except SyntaxException as question_error: # logmessage("SyntaxException") - docassemble.base.functions.reset_context() + reset_context() the_question = None try: the_question = question except: pass if the_question is not None: - raise DASourceError(str(qError) + "\n\n" + str(self.idebug(self.data_for_debug))) - raise DASourceError("no question available in askfor: " + str(qError)) - except CompileException as qError: + raise DASourceError(str(question_error) + "\n\n" + str(self.idebug(self.data_for_debug))) + raise DASourceError("no question available in askfor: " + str(question_error)) + except CompileException as question_error: # logmessage("CompileException") - docassemble.base.functions.reset_context() + reset_context() the_question = None try: the_question = question except: pass if the_question is not None: - raise DASourceError(str(qError) + "\n\n" + str(self.idebug(self.data_for_debug))) - raise DASourceError("no question available in askfor: " + str(qError)) - # except SendFileError as qError: + raise DASourceError(str(question_error) + "\n\n" + str(self.idebug(self.data_for_debug))) + raise DASourceError("no question available in askfor: " + str(question_error)) + # except SendFileError as question_error: # # logmessage("Trapped SendFileError2") # question_data = {'extras': {}} - # if hasattr(qError, 'filename') and qError.filename is not None: - # question_data['response filename'] = qError.filename - # if hasattr(qError, 'content_type') and qError.content_type: - # question_data['content type'] = qError.content_type + # if hasattr(question_error, 'filename') and question_error.filename is not None: + # question_data['response filename'] = question_error.filename + # if hasattr(question_error, 'content_type') and question_error.content_type: + # question_data['content type'] = question_error.content_type # new_interview_source = InterviewSourceString(content='') - # new_interview = new_interview_source.get_interview() + # new_interview = Interview(source=new_interview_source) # new_question = Question(question_data, new_interview, source=new_interview_source, package=self.source.package) # new_question.name = "Question_Temp" # return new_question.ask(user_dict, old_user_dict, 'None', [], None, None) - if 'forgive_missing_question' in docassemble.base.functions.this_thread.misc and origMissingVariable in docassemble.base.functions.this_thread.misc['forgive_missing_question']: - docassemble.base.functions.pop_current_variable() - docassemble.base.functions.pop_event_stack(origMissingVariable) - return {'type': 'continue', 'sought': missing_var, 'orig_sought': origMissingVariable} - raise DAErrorMissingVariable("Interview has an error. There was a reference to a variable '" + origMissingVariable + "' that could not be found in the question file (for language '" + str(language) + "') or in any of the files incorporated by reference into the question file.", variable=origMissingVariable) + if 'forgive_missing_question' in this_thread.misc and orig_missing_variable in this_thread.misc['forgive_missing_question']: + pop_current_variable() + pop_event_stack(orig_missing_variable) + return {'type': 'continue', 'sought': missing_var, 'orig_sought': orig_missing_variable} + raise DAErrorMissingVariable("Interview has an error. There was a reference to a variable '" + orig_missing_variable + "' that could not be found in the question file (for language '" + str(language) + "') or in any of the files incorporated by reference into the question file.", variable=orig_missing_variable) def substitute_vars(var, is_generic, the_x, iterators, last_only=False): @@ -10242,14 +10001,6 @@ def process_selections(data, exclude=None): return result -def extract_missing_name(the_error): - # logmessage("extract_missing_name: string was " + str(string)) - m = nameerror_match.search(str(the_error)) - if m: - return m.group(1) - raise the_error - - def auto_determine_type(field_info, the_value=None): types = {} if 'selections' in field_info: @@ -10408,7 +10159,7 @@ def exec_with_trap(the_question, the_dict, old_variable=None): raise except: cl, exc, tb = sys.exc_info() - exc.user_dict = docassemble.base.functions.serializable_dict(the_dict) + exc.user_dict = serializable_dict(the_dict) if len(traceback.extract_tb(tb)) == 2: line_with_error = traceback.extract_tb(tb)[-1][1] if isinstance(line_with_error, int) and line_with_error > 0 and hasattr(the_question, 'sourcecode'): @@ -10419,8 +10170,8 @@ def exec_with_trap(the_question, the_dict, old_variable=None): del tb raise -ok_outside_string = string.ascii_letters + string.digits + '.[]_' -ok_inside_string = string.ascii_letters + string.digits + string.punctuation + " " +OK_OUTSIDE_STRING = string.ascii_letters + string.digits + '.[]_' +OK_INSIDE_STRING = string.ascii_letters + string.digits + string.punctuation + " " def parse_var_name(var): @@ -10464,7 +10215,7 @@ def parse_var_name(var): the_quote = char else: if not (in_quote or in_bracket): - if char not in ok_outside_string: + if char not in OK_OUTSIDE_STRING: return {'valid': False, 'reason': 'invalid character in variable name'} if cur_pos == 0: if char in string.digits or char == '.': @@ -10473,7 +10224,7 @@ def parse_var_name(var): if var[cur_pos - 1] == '.' and char in string.digits: return {'valid': False, 'reason': 'attribute starts with digit'} if in_quote: - if char not in ok_inside_string: + if char not in OK_INSIDE_STRING: return {'valid': False, 'reason': 'invalid character in string'} else: if char == '.': @@ -10502,522 +10253,6 @@ def parse_var_name(var): return {'valid': True, 'objects': objects, 'bracket_objects': bracket_objects, 'final_parts': final_parts} -class DAExtension(Extension): - - def parse(self, parser): - raise NotImplementedError() - - def filter_stream(self, stream): - # in_var = False - met_pipe = False - for token in stream: - if token.type == 'variable_begin': - # in_var = True - met_pipe = False - if token.type == 'variable_end': - # in_var = False - if not met_pipe: - yield Token(token.lineno, 'pipe', None) - yield Token(token.lineno, 'name', 'ampersand_filter') - # if in_var and token.type == 'pipe': - # met_pipe = True - yield token - - -class DAEnvironment(Environment): - - def from_string(self, source, **kwargs): # pylint: disable=arguments-differ - source = re.sub(r'({[\%\{].*?[\%\}]})', fix_quotes, source) - return super().from_string(source, **kwargs) - - def getitem(self, obj, argument): - try: - return obj[argument] - except (DAAttributeError, DAIndexError) as err: - varname = extract_missing_name(err) - if 'pending_error' in docassemble.base.functions.this_thread.misc: - del docassemble.base.functions.this_thread.misc['pending_error'] - return self.undefined(obj=missing, name=varname) - except (AttributeError, TypeError, LookupError): - if 'pending_error' in docassemble.base.functions.this_thread.misc: - del docassemble.base.functions.this_thread.misc['pending_error'] - return self.undefined(obj=obj, name=argument, accesstype='item') - - def getattr(self, obj, attribute): - try: - return getattr(obj, attribute) - except DAAttributeError as err: - if 'pending_error' in docassemble.base.functions.this_thread.misc: - del docassemble.base.functions.this_thread.misc['pending_error'] - varname = extract_missing_name(err) - return self.undefined(obj=missing, name=varname) - except AttributeError: - if 'pending_error' in docassemble.base.functions.this_thread.misc: - del docassemble.base.functions.this_thread.misc['pending_error'] - return self.undefined(obj=obj, name=attribute, accesstype='attribute') - - -def ampersand_filter(value): - if value.__class__.__name__ in ('DAFile', 'DALink', 'DAStaticFile', 'DAFileCollection', 'DAFileList'): - return value - if value.__class__.__name__ in ('CustomInlineImage', 'InlineImage', 'RichText', 'Listing', 'Document', 'Subdoc', 'DALazyTemplate', 'Markup'): - return str(value) - if isinstance(value, (int, bool, float, NoneType)): - return value - if not isinstance(value, str): - value = str(value) - value = docassemble.base.file_docx.sanitize_xml(value) - if '' in value or '' in value: - return re.sub(r'&(?!#?[0-9A-Za-z]+;)', '&', value) - for auto_filter in docassemble.base.functions.this_thread.misc.get('auto jinja filter', []): - value = auto_filter(value) - return re.sub(r'>', '>', re.sub(r'<', '<', re.sub(r'&(?!#?[0-9A-Za-z]+;)', '&', value))) - - -class DAStrictUndefined(StrictUndefined): - __slots__ = ('_undefined_type',) - - def __init__(self, hint=None, obj=missing, name=None, exc=UndefinedError, accesstype=None): # pylint: disable=super-init-not-called - self._undefined_hint = hint - self._undefined_obj = obj - self._undefined_name = name - self._undefined_exception = exc - self._undefined_type = accesstype - - @internalcode - def __getattr__(self, name): - if name[:2] == '__': - raise AttributeError(name) - return self._fail_with_undefined_error(attribute=True) - - @internalcode - def __getitem__(self, index): - if index[:2] == '__': - raise IndexError(index) - return self._fail_with_undefined_error(item=True) - - @internalcode - def _fail_with_undefined_error(self, *args, **kwargs): - if self._undefined_obj is missing: - hint = "'%s' is undefined" % self._undefined_name - elif self._undefined_type == 'item' and hasattr(self._undefined_obj, 'instanceName'): - hint = "'%s[%r]' is undefined" % ( - self._undefined_obj.instanceName, - self._undefined_name - ) - elif 'attribute' in kwargs or self._undefined_type == 'attribute': - if hasattr(self._undefined_obj, 'instanceName'): - hint = "'%s.%s' is undefined" % ( - self._undefined_obj.instanceName, - self._undefined_name - ) - else: - hint = '%r has no attribute %r' % ( - object_type_repr(self._undefined_obj), - self._undefined_name - ) - else: - if hasattr(self._undefined_obj, 'instanceName'): - hint = "'%s[%r]' is undefined" % ( - self._undefined_obj.instanceName, - self._undefined_name - ) - else: - hint = '%s has no element %r' % ( - object_type_repr(self._undefined_obj), - self._undefined_name - ) - raise self._undefined_exception(hint) - __add__ = __radd__ = __mul__ = __rmul__ = __div__ = __rdiv__ = \ - __truediv__ = __rtruediv__ = __floordiv__ = __rfloordiv__ = \ - __mod__ = __rmod__ = __pos__ = __neg__ = __call__ = \ - __lt__ = __le__ = __gt__ = __ge__ = __int__ = \ - __float__ = __complex__ = __pow__ = __rpow__ = __sub__ = \ - __rsub__ = __iter__ = __str__ = __len__ = __nonzero__ = __eq__ = \ - __ne__ = __bool__ = __hash__ = _fail_with_undefined_error - - -class DASkipUndefined(ChainableUndefined): - """Undefined handler for Jinja2 exceptions that allows rendering most - templates that have undefined variables. It will not fix all broken - templates. For example, if the missing variable is used in a complex - mathematical expression it may still break (but expressions with only two - elements should render as ''). - """ - - def __init__(self, *pargs, **kwargs): # pylint: disable=super-init-not-called - # Handle the way Docassemble DAEnvironment triggers attribute errors - pass - - def __str__(self) -> str: - return '' - - def __call__(self, *pargs, **kwargs) -> "DASkipUndefined": - return self - - __getitem__ = __getattr__ = __call__ - - def __eq__(self, *pargs) -> bool: - return False - - # need to return a bool type - __bool__ = __ne__ = __le__ = __lt__ = __gt__ = __ge__ = __nonzero__ = __eq__ - - # let undefined variables work in for loops - - def __iter__(self, *pargs) -> "DASkipUndefined": - return self - - def __next__(self, *pargs) -> None: - raise StopIteration - - # need to return an int type - - def __int__(self, *pargs) -> int: - return 0 - - __len__ = __int__ - - # need to return a float type - - def __float__(self, *pargs) -> float: - return 0.0 - - # need to return complex type - - def __complex__(self, *pargs) -> complex: - return 0j - - def __add__(self, *pargs, **kwargs) -> str: - return self.__str__() - - # type can be anything. we want it to work with `str()` function though - # and we do not want to silently give wrong math results. - # note that this means 1 + (undefined) or (undefined) + 1 will work but not 1 + (undefined) + 1 - __radd__ = __mul__ = __rmul__ = __div__ = __rdiv__ = \ - __truediv__ = __rtruediv__ = __floordiv__ = __rfloordiv__ = \ - __mod__ = __rmod__ = __pos__ = __neg__ = __pow__ = __rpow__ = \ - __sub__ = __rsub__ = __hash__ = __add__ - - -def mygetattr(y, attr): - for attribute in attr.split('.'): - y = getattr(y, attribute) - return y - - -def str_or_original(y, case_sensitive): - if case_sensitive: - if hasattr(y, 'instanceName'): - if y.__class__.__name__ in ('Value', 'PeriodicValue'): - return y.amount() - return str(y) - return y - if hasattr(y, 'instanceName'): - if y.__class__.__name__ in ('Value', 'PeriodicValue'): - return y.amount() - return str(y).lower() - try: - return y.lower() - except: - return y - - -def dictsort_filter(dictionary, case_sensitive=False, by='key', reverse=False): - if by == 'value': - return sorted(dictionary.items(), key=lambda y: str_or_original(y[1], case_sensitive), reverse=reverse) - return sorted(dictionary.items(), key=lambda y: str_or_original(y[0], case_sensitive), reverse=reverse) - - -def sort_filter(the_array, reverse=False, case_sensitive=False, attribute=None): - if attribute is None: - if not case_sensitive: - def key_func(y): - return str_or_original(y, case_sensitive) - else: - key_func = None - else: - if isinstance(attribute, list): - attributes = [str(y).strip() for y in attribute] - else: - attributes = [y.strip() for y in str(attribute).split(',')] - def key_func(y): - return [str_or_original(mygetattr(y, attribute), case_sensitive) for attribute in attributes] - return sorted(the_array, key=key_func, reverse=reverse) - -_GroupTuple = namedtuple('_GroupTuple', ['grouper', 'list']) -_GroupTuple.__repr__ = tuple.__repr__ -_GroupTuple.__str__ = tuple.__str__ - - -def groupby_filter(the_array, attr_name): - - def func(y): - return mygetattr(y, attr_name) - return [_GroupTuple(key, list(values)) for key, values in groupby(sorted(the_array, key=func), func)] - - -def max_filter(the_array, case_sensitive=False, attribute=None): - it = iter(the_array) - try: - first = next(it) - except StopIteration: - raise DAError("max: list was empty") - if attribute: - def key_func(y): - return str_or_original(mygetattr(y, attribute), case_sensitive=case_sensitive) - else: - def key_func(y): - return str_or_original(y, case_sensitive=case_sensitive) - return max(chain([first], it), key=key_func) - - -def min_filter(the_array, case_sensitive=False, attribute=None): - it = iter(the_array) - try: - first = next(it) - except StopIteration: - raise DAError("min: list was empty") - if attribute: - def key_func(y): - return str_or_original(mygetattr(y, attribute), case_sensitive=case_sensitive) - else: - def key_func(y): - return str_or_original(y, case_sensitive=case_sensitive) - return min(chain([first], it), key=key_func) - - -def sum_filter(the_array, attribute=None, start=0): - if attribute is not None: - the_array = [mygetattr(y, attribute) for y in the_array] - return sum(the_array, start) - - -def unique_filter(the_array, case_sensitive=False, attribute=None): - seen = set() - if attribute is None: - for item in the_array: - new_item = str_or_original(item, case_sensitive) - if new_item not in seen: - seen.add(new_item) - yield item - else: - for item in the_array: - new_item = str_or_original(mygetattr(item, attribute), case_sensitive) - if new_item not in seen: - seen.add(new_item) - yield mygetattr(item, attribute) - - -def join_filter(the_array, d="", attribute=None): - if attribute is not None: - return d.join([str(mygetattr(y, attribute)) for y in the_array]) - return d.join([str(y) for y in the_array]) - - -def attr_filter(var, attr_name): - return mygetattr(var, attr_name) - - -def selectattr_filter(*pargs, **kwargs): - if len(pargs) > 2: - the_array = pargs[0] - attr_name = pargs[1] - func_name = pargs[2] - env = custom_jinja_env() - def func(item): - return env.call_test(func_name, item, pargs[3:], kwargs) - for item in the_array: - if func(mygetattr(item, attr_name)): - yield item - else: - for item in pargs[0]: - if mygetattr(item, pargs[1]): - yield item - - -def rejectattr_filter(*pargs, **kwargs): - if len(pargs) > 2: - the_array = pargs[0] - attr_name = pargs[1] - func_name = pargs[2] - env = custom_jinja_env() - def func(item): - return env.call_test(func_name, item, pargs[3:], kwargs) - for item in the_array: - if not func(mygetattr(item, attr_name)): - yield item - else: - for item in pargs[0]: - if not mygetattr(item, pargs[1]): - yield item - - -def chain_filter(*pargs, **kwargs): # pylint: disable=unused-argument - the_list = [] - for parg in pargs: - if isinstance(parg, str): - the_list.append(parg) - elif (hasattr(parg, 'instanceName') and hasattr(parg, 'elements')): - if isinstance(parg.elements, dict): - for sub_parg in parg.values(): - the_list.append(sub_parg) - else: - for sub_parg in parg: - the_list.append(sub_parg) - elif isinstance(parg, abc.Iterable): - for sub_parg in parg: - the_list.append(sub_parg) - else: - the_list.append(parg) - return chain(*the_list) - - -def map_filter(*pargs, **kwargs): - if len(pargs) >= 2: - the_array = pargs[0] - the_filter = pargs[1] - env = custom_jinja_env() - if the_filter not in env.filters: - raise DAError('filter passed to map() does not exist') - for item in the_array: - yield env.call_filter(the_filter, item, pargs[2:], kwargs) - else: - if 'attribute' in kwargs: - if 'default' in kwargs: - for item in pargs[0]: - yield mygetattr(item, kwargs['attribute'], kwargs['default']) - else: - for item in pargs[0]: - yield mygetattr(item, kwargs['attribute']) - elif 'index' in kwargs: - if 'default' in kwargs: - for item in pargs[0]: - yield item.get(kwargs['index'], kwargs['default']) - else: - for item in pargs[0]: - yield item[kwargs['index']] - elif 'function' in kwargs: - the_kwargs = kwargs.get('kwargs', {}) - the_pargs = kwargs.get('pargs', []) - if not isinstance(the_kwargs, dict): - raise DAError('kwargs passed to map() must be a dictionary') - if not isinstance(the_pargs, list): - raise DAError('pargs passed to map() must be a list') - for item in pargs[0]: - yield kwargs['function'](item, *the_pargs, **the_kwargs) - else: - raise DAError('map() must refer to a function, index, attribute, or filter') - - -def markdown_filter(text): - return docassemble.base.file_docx.markdown_to_docx(text, docassemble.base.functions.this_thread.current_question, docassemble.base.functions.this_thread.misc.get('docx_template', None)) - - -def inline_markdown_filter(text): - return docassemble.base.file_docx.inline_markdown_to_docx(text, docassemble.base.functions.this_thread.current_question, docassemble.base.functions.this_thread.misc.get('docx_template', None)) - - -def get_builtin_jinja_filters(): - return { - 'ampersand_filter': ampersand_filter, - 'markdown': markdown_filter, - 'add_separators': docassemble.base.functions.add_separators, - 'inline_markdown': inline_markdown_filter, - 'paragraphs': docassemble.base.functions.single_to_double_newlines, - 'manual_line_breaks': docassemble.base.functions.manual_line_breaks, - 'RichText': docassemble.base.file_docx.RichText, - 'groupby': groupby_filter, - 'max': max_filter, - 'min': min_filter, - 'sum': sum_filter, - 'unique': unique_filter, - 'join': join_filter, - 'attr': attr_filter, - 'selectattr': selectattr_filter, - 'rejectattr': rejectattr_filter, - 'sort': sort_filter, - 'dictsort': dictsort_filter, - 'format_date': docassemble.base.util.format_date, - 'format_datetime': docassemble.base.util.format_datetime, - 'format_time': docassemble.base.util.format_time, - 'month_of': docassemble.base.util.month_of, - 'year_of': docassemble.base.util.year_of, - 'day_of': docassemble.base.util.day_of, - 'dow_of': docassemble.base.util.dow_of, - 'qr_code': docassemble.base.functions.qr_code, - 'nice_number': docassemble.base.functions.nice_number, - 'ordinal': docassemble.base.functions.ordinal, - 'ordinal_number': docassemble.base.functions.ordinal_number, - 'currency': docassemble.base.functions.currency, - 'comma_list': docassemble.base.functions.comma_list, - 'comma_and_list': docassemble.base.functions.comma_and_list, - 'capitalize': docassemble.base.functions.capitalize, - 'salutation': docassemble.base.functions.salutation, - 'alpha': docassemble.base.functions.alpha, - 'roman': docassemble.base.functions.roman, - 'word': docassemble.base.functions.word, - 'bold': docassemble.base.functions.bold, - 'italic': docassemble.base.functions.italic, - 'title_case': docassemble.base.functions.title_case, - 'single_paragraph': docassemble.base.functions.single_paragraph, - 'phone_number_formatted': docassemble.base.functions.phone_number_formatted, - 'phone_number_in_e164': docassemble.base.functions.phone_number_in_e164, - 'country_name': docassemble.base.functions.country_name, - 'fix_punctuation': docassemble.base.functions.fix_punctuation, - 'redact': docassemble.base.functions.redact, - 'verbatim': docassemble.base.functions.verbatim, - 'map': map_filter, - 'chain': chain_filter, - 'any': any, - 'all': all - } - - -registered_jinja_filters = {} - - -def custom_jinja_env(skip_undefined=False): - if skip_undefined: - env = DAEnvironment(undefined=DASkipUndefined, extensions=[DAExtension]) - else: - env = DAEnvironment(undefined=DAStrictUndefined, extensions=[DAExtension]) - env.filters.update(registered_jinja_filters) - env.filters.update(get_builtin_jinja_filters()) - return env - - -def register_jinja_filter(filter_name, func): - if filter_name in get_builtin_jinja_filters(): - raise DAError("Cannot register filter with same name as built-in filter %s" % filter_name) - registered_jinja_filters[filter_name] = func - - -def get_docx_variables(the_path): - names = set() - if not os.path.isfile(the_path): - raise DASourceError("Missing docx template file " + os.path.basename(the_path)) - try: - docx_template = DocxTemplate(the_path) - docx_template.render_init() - the_env = custom_jinja_env() - the_xml = docx_template.get_xml() - the_xml = re.sub(r'])', r'\n