From aa2df1935e1117b86da9f95d8afe6a8a5f850014 Mon Sep 17 00:00:00 2001 From: Shreyas Mayya <2022smayya@tjhsst.edu> Date: Sat, 5 Feb 2022 23:13:01 -0500 Subject: [PATCH] feat(destinations): use mistune for Markdown parsing --- Pipfile | 3 +- Pipfile.lock | 184 ++++++------------ scripts/check.sh | 2 +- scripts/format.sh | 2 +- .../destinations/templatetags/markdown.py | 23 ++- .../destinations/templatetags/sanitize.py | 19 -- .../templatetags/strikethrough.py | 39 ---- tjdests/static/bios.css | 138 ++++++------- .../templates/destinations/student_list.html | 3 +- 9 files changed, 154 insertions(+), 259 deletions(-) delete mode 100644 tjdests/apps/destinations/templatetags/sanitize.py delete mode 100644 tjdests/apps/destinations/templatetags/strikethrough.py diff --git a/Pipfile b/Pipfile index a2d6c22..01eb9b8 100644 --- a/Pipfile +++ b/Pipfile @@ -4,7 +4,6 @@ verify_ssl = true name = "pypi" [packages] -bleach = "~=4.1.0" crispy-bootstrap5 = "~=0.6" django = "~=3.2.11" django-bootstrap-pagination = "~=1.7.1" @@ -12,7 +11,7 @@ django-crispy-forms = "~=1.13.0" django-extensions = "~=3.1.5" gunicorn = "~=20.1.0" ipython = "~=7.31.0" -markdown = "~=3.3.6" +mistune = "~=2.0.2" psycopg2 = "~=2.9.3" pygments = "~=2.10.0" social-auth-app-django = "~=5.0.0" diff --git a/Pipfile.lock b/Pipfile.lock index 24997d1..4ed6808 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "2c7f32efd684e3a0b5cab04853273c113b3be7dc08e37da0afac86e28de7bbad" + "sha256": "227de253d05fb4f8b4d6b6c577312f4de5e0f4238deeb329e62c2451e7717a1f" }, "pipfile-spec": 6, "requires": { @@ -31,14 +31,6 @@ ], "version": "==0.2.0" }, - "bleach": { - "hashes": [ - "sha256:0900d8b37eba61a802ee40ac0061f8c2b5dee29c1927dd1d233e075ebf5a71da", - "sha256:4d2651ab93271d1129ac9cbc679f524565cc8a1b791909c4a51eac4446a15994" - ], - "index": "pypi", - "version": "==4.1.0" - }, "certifi": { "hashes": [ "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872", @@ -103,11 +95,11 @@ }, "charset-normalizer": { "hashes": [ - "sha256:876d180e9d7432c5d1dfd4c5d26b72f099d503e8fcc0feb7532c9289be60fcbd", - "sha256:cb957888737fc0bbcd78e3df769addb41fd1ff8cf950dc9e7ad7793f1bf44455" + "sha256:2842d8f5e82a1f6aa437380934d5e1cd4fcf2003b06fed6940769c164a480a45", + "sha256:98398a9d69ee80548c762ba991a4728bfc3836768ed226b3945908d1a688371c" ], "markers": "python_version >= '3'", - "version": "==2.0.10" + "version": "==2.0.11" }, "crispy-bootstrap5": { "hashes": [ @@ -161,11 +153,11 @@ }, "django": { "hashes": [ - "sha256:0a0a37f0b93aef30c4bf3a839c187e1175bcdeb7e177341da0cb7b8194416891", - "sha256:69c94abe5d6b1b088bf475e09b7b74403f943e34da107e798465d2045da27e75" + "sha256:9772e6935703e59e993960832d66a614cf0233a1c5123bc6224ecc6ad69e41e2", + "sha256:9b06c289f9ba3a8abea16c9c9505f25107809fb933676f6c891ded270039d965" ], "index": "pypi", - "version": "==3.2.11" + "version": "==3.2.12" }, "django-bootstrap-pagination": { "hashes": [ @@ -207,14 +199,6 @@ "markers": "python_version >= '3'", "version": "==3.3" }, - "importlib-metadata": { - "hashes": [ - "sha256:899e2a40a8c4a1aec681feef45733de8a6c58f3f6a0dbed2eb6574b4387a77b6", - "sha256:951f0d8a5b7260e9db5e41d429285b5f451e928479f19d80818878527d36e95e" - ], - "markers": "python_version < '3.10'", - "version": "==4.10.1" - }, "ipython": { "hashes": [ "sha256:55df3e0bd0f94e715abd968bedd89d4e8a7bce4bf498fb123fed4f5398fea874", @@ -231,14 +215,6 @@ "markers": "python_version >= '3.6'", "version": "==0.18.1" }, - "markdown": { - "hashes": [ - "sha256:76df8ae32294ec39dcf89340382882dfa12975f87f45c3ed1ecdb1e8cefc7006", - "sha256:9923332318f843411e9932237530df53162e29dc7a4e2b91e35764583c46c9a3" - ], - "index": "pypi", - "version": "==3.3.6" - }, "matplotlib-inline": { "hashes": [ "sha256:a04bfba22e0d1395479f866853ec1ee28eea1485c1d69a6faf00dc3e24ff34ee", @@ -247,6 +223,14 @@ "markers": "python_version >= '3.5'", "version": "==0.1.3" }, + "mistune": { + "hashes": [ + "sha256:6bab6c6abd711c4604206c7d8cad5cd48b28f072b4bb75797d74146ba393a049", + "sha256:6fc88c3cb49dba8b16687b41725e661cf85784c12e8974a29b9d336dd596c3a1" + ], + "index": "pypi", + "version": "==2.0.2" + }, "oauthlib": { "hashes": [ "sha256:23a8208d75b902797ea29fd31fa80a15ed9dc2c6c16fe73f5d346f83f6fa27a2", @@ -255,14 +239,6 @@ "markers": "python_version >= '3.6'", "version": "==3.2.0" }, - "packaging": { - "hashes": [ - "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", - "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" - ], - "markers": "python_version >= '3.6'", - "version": "==21.3" - }, "parso": { "hashes": [ "sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0", @@ -342,14 +318,6 @@ "markers": "python_version >= '3.6'", "version": "==2.3.0" }, - "pyparsing": { - "hashes": [ - "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea", - "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484" - ], - "markers": "python_version >= '3.6'", - "version": "==3.0.7" - }, "python3-openid": { "hashes": [ "sha256:33fbf6928f401e0b790151ed2b5290b02545e8775f982485205a066f874aaeaf", @@ -380,14 +348,6 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.3.1" }, - "six": { - "hashes": [ - "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", - "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.16.0" - }, "social-auth-app-django": { "hashes": [ "sha256:52241a25445a010ab1c108bafff21fc5522d5c8cd0d48a92c39c7371824b065d", @@ -435,13 +395,6 @@ ], "version": "==0.2.5" }, - "webencodings": { - "hashes": [ - "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", - "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" - ], - "version": "==0.5.1" - }, "whitenoise": { "hashes": [ "sha256:d234b871b52271ae7ed6d9da47ffe857c76568f11dd30e28e18c5869dbd11e12", @@ -449,14 +402,6 @@ ], "index": "pypi", "version": "==5.3.0" - }, - "zipp": { - "hashes": [ - "sha256:9f50f446828eb9d45b267433fd3e9da8d801f614129124863f9c51ebceafb87d", - "sha256:b47250dd24f92b7dd6a0a8fc5244da14608f3ca90a5efcd37a3b1642fac9a375" - ], - "markers": "python_version >= '3.7'", - "version": "==3.7.0" } }, "develop": { @@ -502,61 +447,58 @@ }, "coverage": { "hashes": [ - "sha256:012157499ec4f135fc36cd2177e3d1a1840af9b236cbe80e9a5ccfc83d912a69", - "sha256:0a34d313105cdd0d3644c56df2d743fe467270d6ab93b5d4a347eb9fec8924d6", - "sha256:11e61c5548ecf74ea1f8b059730b049871f0e32b74f88bd0d670c20c819ad749", - "sha256:152cc2624381df4e4e604e21bd8e95eb8059535f7b768c1fb8b8ae0b26f47ab0", - "sha256:1b4285fde5286b946835a1a53bba3ad41ef74285ba9e8013e14b5ea93deaeafc", - "sha256:27a94db5dc098c25048b0aca155f5fac674f2cf1b1736c5272ba28ead2fc267e", - "sha256:27ac7cb84538e278e07569ceaaa6f807a029dc194b1c819a9820b9bb5dbf63ab", - "sha256:2a491e159294d756e7fc8462f98175e2d2225e4dbe062cca7d3e0d5a75ba6260", - "sha256:2bc85664b06ba42d14bb74d6ddf19d8bfc520cb660561d2d9ce5786ae72f71b5", - "sha256:32168001f33025fd756884d56d01adebb34e6c8c0b3395ca8584cdcee9c7c9d2", - "sha256:3c4ce3b647bd1792d4394f5690d9df6dc035b00bcdbc5595099c01282a59ae01", - "sha256:433b99f7b0613bdcdc0b00cc3d39ed6d756797e3b078d2c43f8a38288520aec6", - "sha256:4578728c36de2801c1deb1c6b760d31883e62e33f33c7ba8f982e609dc95167d", - "sha256:509c68c3e2015022aeda03b003dd68fa19987cdcf64e9d4edc98db41cfc45d30", - "sha256:51372e24b1f7143ee2df6b45cff6a721f3abe93b1e506196f3ffa4155c2497f7", - "sha256:5d008e0f67ac800b0ca04d7914b8501312c8c6c00ad8c7ba17754609fae1231a", - "sha256:649df3641eb351cdfd0d5533c92fc9df507b6b2bf48a7ef8c71ab63cbc7b5c3c", - "sha256:6e78b1e25e5c5695dea012be473e442f7094d066925604be20b30713dbd47f89", - "sha256:72d9d186508325a456475dd05b1756f9a204c7086b07fffb227ef8cee03b1dc2", - "sha256:7d82c610a2e10372e128023c5baf9ce3d270f3029fe7274ff5bc2897c68f1318", - "sha256:7ee317486593193e066fc5e98ac0ce712178c21529a85c07b7cb978171f25d53", - "sha256:7eed8459a2b81848cafb3280b39d7d49950d5f98e403677941c752e7e7ee47cb", - "sha256:823f9325283dc9565ba0aa2d240471a93ca8999861779b2b6c7aded45b58ee0f", - "sha256:85c5fc9029043cf8b07f73fbb0a7ab6d3b717510c3b5642b77058ea55d7cacde", - "sha256:86c91c511853dfda81c2cf2360502cb72783f4b7cebabef27869f00cbe1db07d", - "sha256:8e0c3525b1a182c8ffc9bca7e56b521e0c2b8b3e82f033c8e16d6d721f1b54d6", - "sha256:987a84ff98a309994ca77ed3cc4b92424f824278e48e4bf7d1bb79a63cfe2099", - "sha256:9ed3244b415725f08ca3bdf02ed681089fd95e9465099a21c8e2d9c5d6ca2606", - "sha256:a189036c50dcd56100746139a459f0d27540fef95b09aba03e786540b8feaa5f", - "sha256:a4748349734110fd32d46ff8897b561e6300d8989a494ad5a0a2e4f0ca974fc7", - "sha256:a5d79c9af3f410a2b5acad91258b4ae179ee9c83897eb9de69151b179b0227f5", - "sha256:a7596aa2f2b8fa5604129cfc9a27ad9beec0a96f18078cb424d029fdd707468d", - "sha256:ab4fc4b866b279740e0d917402f0e9a08683e002f43fa408e9655818ed392196", - "sha256:bde4aeabc0d1b2e52c4036c54440b1ad05beeca8113f47aceb4998bb7471e2c2", - "sha256:c72bb4679283c6737f452eeb9b2a0e570acaef2197ad255fb20162adc80bea76", - "sha256:c8582e9280f8d0f38114fe95a92ae8d0790b56b099d728cc4f8a2e14b1c4a18c", - "sha256:ca29c352389ea27a24c79acd117abdd8a865c6eb01576b6f0990cd9a4e9c9f48", - "sha256:ce443a3e6df90d692c38762f108fc4c88314bf477689f04de76b3f252e7a351c", - "sha256:d1675db48490e5fa0b300f6329ecb8a9a37c29b9ab64fa9c964d34111788ca2d", - "sha256:da1a428bdbe71f9a8c270c7baab29e9552ac9d0e0cba5e7e9a4c9ee6465d258d", - "sha256:e4ff163602c5c77e7bb4ea81ba5d3b793b4419f8acd296aae149370902cf4e92", - "sha256:e67ccd53da5958ea1ec833a160b96357f90859c220a00150de011b787c27b98d", - "sha256:e8071e7d9ba9f457fc674afc3de054450be2c9b195c470147fbbc082468d8ff7", - "sha256:fff16a30fdf57b214778eff86391301c4509e327a65b877862f7c929f10a4253" + "sha256:1245ab82e8554fa88c4b2ab1e098ae051faac5af829efdcf2ce6b34dccd5567c", + "sha256:1bc6d709939ff262fd1432f03f080c5042dc6508b6e0d3d20e61dd045456a1a0", + "sha256:25e73d4c81efa8ea3785274a2f7f3bfbbeccb6fcba2a0bdd3be9223371c37554", + "sha256:276b13cc085474e482566c477c25ed66a097b44c6e77132f3304ac0b039f83eb", + "sha256:2aed4761809640f02e44e16b8b32c1a5dee5e80ea30a0ff0912158bde9c501f2", + "sha256:2dd70a167843b4b4b2630c0c56f1b586fe965b4f8ac5da05b6690344fd065c6b", + "sha256:352c68e233409c31048a3725c446a9e48bbff36e39db92774d4f2380d630d8f8", + "sha256:3f2b05757c92ad96b33dbf8e8ec8d4ccb9af6ae3c9e9bd141c7cc44d20c6bcba", + "sha256:448d7bde7ceb6c69e08474c2ddbc5b4cd13c9e4aa4a717467f716b5fc938a734", + "sha256:463e52616ea687fd323888e86bf25e864a3cc6335a043fad6bbb037dbf49bbe2", + "sha256:482fb42eea6164894ff82abbcf33d526362de5d1a7ed25af7ecbdddd28fc124f", + "sha256:56c4a409381ddd7bbff134e9756077860d4e8a583d310a6f38a2315b9ce301d0", + "sha256:56d296cbc8254a7dffdd7bcc2eb70be5a233aae7c01856d2d936f5ac4e8ac1f1", + "sha256:5e15d424b8153756b7c903bde6d4610be0c3daca3986173c18dd5c1a1625e4cd", + "sha256:618eeba986cea7f621d8607ee378ecc8c2504b98b3fdc4952b30fe3578304687", + "sha256:61d47a897c1e91f33f177c21de897267b38fbb45f2cd8e22a710bcef1df09ac1", + "sha256:621f6ea7260ea2ffdaec64fe5cb521669984f567b66f62f81445221d4754df4c", + "sha256:6a5cdc3adb4f8bb8d8f5e64c2e9e282bc12980ef055ec6da59db562ee9bdfefa", + "sha256:6c3f6158b02ac403868eea390930ae64e9a9a2a5bbfafefbb920d29258d9f2f8", + "sha256:704f89b87c4f4737da2860695a18c852b78ec7279b24eedacab10b29067d3a38", + "sha256:72128176fea72012063200b7b395ed8a57849282b207321124d7ff14e26988e8", + "sha256:78fbb2be068a13a5d99dce9e1e7d168db880870f7bc73f876152130575bd6167", + "sha256:7bff3a98f63b47464480de1b5bdd80c8fade0ba2832c9381253c9b74c4153c27", + "sha256:84f2436d6742c01136dd940ee158bfc7cf5ced3da7e4c949662b8703b5cd8145", + "sha256:9976fb0a5709988778ac9bc44f3d50fccd989987876dfd7716dee28beed0a9fa", + "sha256:9ad0a117b8dc2061ce9461ea4c1b4799e55edceb236522c5b8f958ce9ed8fa9a", + "sha256:9e3dd806f34de38d4c01416344e98eab2437ac450b3ae39c62a0ede2f8b5e4ed", + "sha256:9eb494070aa060ceba6e4bbf44c1bc5fa97bfb883a0d9b0c9049415f9e944793", + "sha256:9fde6b90889522c220dd56a670102ceef24955d994ff7af2cb786b4ba8fe11e4", + "sha256:9fff3ff052922cb99f9e52f63f985d4f7a54f6b94287463bc66b7cdf3eb41217", + "sha256:a06c358f4aed05fa1099c39decc8022261bb07dfadc127c08cfbd1391b09689e", + "sha256:a4f923b9ab265136e57cc14794a15b9dcea07a9c578609cd5dbbfff28a0d15e6", + "sha256:c5b81fb37db76ebea79aa963b76d96ff854e7662921ce742293463635a87a78d", + "sha256:d5ed164af5c9078596cfc40b078c3b337911190d3faeac830c3f1274f26b8320", + "sha256:d651fde74a4d3122e5562705824507e2f5b2d3d57557f1916c4b27635f8fbe3f", + "sha256:de73fca6fb403dd72d4da517cfc49fcf791f74eee697d3219f6be29adf5af6ce", + "sha256:e647a0be741edbb529a72644e999acb09f2ad60465f80757da183528941ff975", + "sha256:e92c7a5f7d62edff50f60a045dc9542bf939758c95b2fcd686175dd10ce0ed10", + "sha256:eeffd96882d8c06d31b65dddcf51db7c612547babc1c4c5db6a011abe9798525", + "sha256:f5a4551dfd09c3bd12fca8144d47fe7745275adf3229b7223c2f9e29a975ebda", + "sha256:fac0bcc5b7e8169bffa87f0dcc24435446d329cbc2b5486d155c2e0f3b493ae1" ], "index": "pypi", - "version": "==6.3" + "version": "==6.3.1" }, "django": { "hashes": [ - "sha256:0a0a37f0b93aef30c4bf3a839c187e1175bcdeb7e177341da0cb7b8194416891", - "sha256:69c94abe5d6b1b088bf475e09b7b74403f943e34da107e798465d2045da27e75" + "sha256:9772e6935703e59e993960832d66a614cf0233a1c5123bc6224ecc6ad69e41e2", + "sha256:9b06c289f9ba3a8abea16c9c9505f25107809fb933676f6c891ded270039d965" ], "index": "pypi", - "version": "==3.2.11" + "version": "==3.2.12" }, "django-stubs": { "hashes": [ @@ -846,10 +788,10 @@ }, "types-pyyaml": { "hashes": [ - "sha256:6ea4eefa8579e0ce022f785a62de2bcd647fad4a81df5cf946fd67e4b059920b", - "sha256:8b50294b55a9db89498cdc5a65b1b4545112b6cd1cf4465bd693d828b0282a17" + "sha256:6252f62d785e730e454dfa0c9f0fb99d8dae254c5c3c686903cf878ea27c04b7", + "sha256:693b01c713464a6851f36ff41077f8adbc6e355eda929addfb4a97208aea9b4b" ], - "version": "==6.0.3" + "version": "==6.0.4" }, "typing-extensions": { "hashes": [ diff --git a/scripts/check.sh b/scripts/check.sh index 5de221c..02ee486 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -23,7 +23,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -cd "$(dirname -- "$(dirname -- "$(readlink -f "$0")")")" +cd "$(dirname -- "$(dirname -- "$(readlink -f "$0")")")" || exit for cmd in flake8 isort mypy pylint; do if [[ ! -x "$(which "$cmd")" ]]; then diff --git a/scripts/format.sh b/scripts/format.sh index 1cb2e78..67d46a9 100755 --- a/scripts/format.sh +++ b/scripts/format.sh @@ -23,7 +23,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -cd "$(dirname -- "$(dirname -- "$(readlink -f "$0")")")" +cd "$(dirname -- "$(dirname -- "$(readlink -f "$0")")")" || exit for cmd in black autopep8 isort; do if [[ ! -x "$(which "$cmd")" ]]; then diff --git a/tjdests/apps/destinations/templatetags/markdown.py b/tjdests/apps/destinations/templatetags/markdown.py index c6ae66f..36b4b0f 100644 --- a/tjdests/apps/destinations/templatetags/markdown.py +++ b/tjdests/apps/destinations/templatetags/markdown.py @@ -1,15 +1,28 @@ -from markdown import markdown +import mistune +from pygments import highlight +from pygments.formatters import html +from pygments.lexers import get_lexer_by_name from django import template -from .strikethrough import StrikethroughExtension - register = template.Library() +class HighlightRenderer(mistune.HTMLRenderer): + def block_code(self, code, info=None): + if info: + lexer = get_lexer_by_name(info, stripall=True) + formatter = html.HtmlFormatter() + return highlight(code, lexer, formatter) + return "
" + mistune.escape(code) + "
"
+
+
@register.filter(name="markdown")
def convert_markdown(text: str):
"""Convert text to markdown HTML."""
- return markdown(
- text, extensions=["extra", "codehilite", "smarty", StrikethroughExtension()]
+ markdown = mistune.create_markdown(
+ renderer=HighlightRenderer(),
+ plugins=["footnotes", "strikethrough", "table"],
+ escape=False,
)
+ return markdown(text)
diff --git a/tjdests/apps/destinations/templatetags/sanitize.py b/tjdests/apps/destinations/templatetags/sanitize.py
deleted file mode 100644
index e04f277..0000000
--- a/tjdests/apps/destinations/templatetags/sanitize.py
+++ /dev/null
@@ -1,19 +0,0 @@
-from bleach import clean
-from bleach.sanitizer import ALLOWED_TAGS
-
-from django import template
-
-register = template.Library()
-
-tags = (
- ALLOWED_TAGS
- + ["h" + str(i) for i in range(1, 7)]
- + ["div", "p", "pre", "span", "s"]
-)
-attrs = {"*": ["class"]}
-
-
-@register.filter(name="sanitize")
-def convert_markdown(text: str):
- """Sanitize HTML (removing potential XSS attacks)."""
- return clean(text, tags=tags, attributes=attrs)
diff --git a/tjdests/apps/destinations/templatetags/strikethrough.py b/tjdests/apps/destinations/templatetags/strikethrough.py
deleted file mode 100644
index ab1df5f..0000000
--- a/tjdests/apps/destinations/templatetags/strikethrough.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# -*- coding: utf-8 -*-
-# pylint: skip-file
-"""Markdown Strikethrough Extension
-Extends the Python-Markdown library to support strikethrough text.
-Given the text:
- The molecular composition of water is ~~HCl~~.
-Will output:
- The molecular composition of water is HCl.