NIST researchers warn of top AI security threats | StateScoop<\/title> <meta name=\"description\" content=\"State and local governments are among the organizations threatened by various exploits against AI systems, according to a recent paper.\"> <link rel=\"canonical\" href=\"https:\/\/statescoop.com\/nist-security-threats-ai-state-local\/\"> <meta property=\"og:locale\" content=\"en_US\"> <meta property=\"og:type\" content=\"article\"> <meta property=\"og:title\" content=\"NIST researchers warn of top AI security threats | StateScoop\"> <meta property=\"og:description\" content=\"State and local governments are among the organizations threatened by various exploits against AI systems, according to a recent paper.\"> <meta property=\"og:url\" content=\"https:\/\/statescoop.com\/nist-security-threats-ai-state-local\/\"> <meta property=\"og:site_name\" content=\"StateScoop\"> <meta property=\"article:published_time\" content=\"2024-01-11T19:04:26+00:00\"> <meta property=\"article:modified_time\" content=\"2024-01-12T19:12:32+00:00\"> <meta property=\"og:image\" content=\"https:\/\/ddi.mohflo.net\/wp-content\/uploads\/2024\/01\/nist-researchers-warn-of-top-ai-security-threats-2.jpg\"> <meta property=\"og:image:width\" content=\"1802\"> <meta property=\"og:image:height\" content=\"1014\"> <meta property=\"og:image:type\" content=\"image\/jpeg\"> <meta name=\"author\" content=\"sfoxsowell\"> <meta name=\"twitter:card\" content=\"summary_large_image\">  <link rel=\"dns-prefetch\" href=\"\/\/securepubads.g.doubleclick.net\">\n<link rel=\"dns-prefetch\" href=\"\/\/use.typekit.net\">\n<link rel=\"alternate\" type=\"application\/rss+xml\" title=\"StateScoop \u00bb Feed\" href=\"https:\/\/statescoop.com\/feed\/\">\n<link rel=\"alternate\" type=\"application\/rss+xml\" title=\"StateScoop \u00bb Comments Feed\" href=\"https:\/\/statescoop.com\/comments\/feed\/\"> <link rel=\"stylesheet\" id=\"all-css-2\" href=\"https:\/\/statescoop.com\/wp-includes\/css\/dist\/block-library\/style.min.css?m=1701905043g\" type=\"text\/css\" media=\"all\"> <link rel=\"stylesheet\" id=\"all-css-6\" href=\"https:\/\/statescoop.com\/wp-content\/mu-plugins\/search\/elasticpress-next\/dist\/css\/related-posts-block-styles.min.css?m=1704748048g\" type=\"text\/css\" media=\"all\"> <link rel=\"stylesheet\" id=\"all-css-8\" href=\"https:\/\/statescoop.com\/wp-content\/themes\/scoopnewsgroup\/dist\/css\/frontend.css?m=1704975497g\" type=\"text\/css\" media=\"all\">\n<link rel=\"stylesheet\" id=\"typekit-css\" href=\"https:\/\/use.typekit.net\/itk2qbh.css?ver=7dab012cdc88b5676610\" media=\"all\"> <link rel=\"https:\/\/api.w.org\/\" href=\"https:\/\/statescoop.com\/wp-json\/\"><link rel=\"alternate\" type=\"application\/json\" href=\"https:\/\/statescoop.com\/wp-json\/wp\/v2\/posts\/61230\"><link rel=\"EditURI\" type=\"application\/rsd+xml\" title=\"RSD\" href=\"https:\/\/statescoop.com\/xmlrpc.php?rsd\">\n<meta name=\"generator\" content=\"WordPress 6.4.2\">\n<link rel=\"shortlink\" href=\"https:\/\/statescoop.com\/?p=61230\">\n<link rel=\"alternate\" type=\"application\/json+oembed\" href=\"https:\/\/statescoop.com\/wp-json\/oembed\/1.0\/embed?url=https%3A%2F%2Fstatescoop.com%2Fnist-security-threats-ai-state-local%2F\">\n<link rel=\"alternate\" type=\"text\/xml+oembed\" href=\"https:\/\/statescoop.com\/wp-json\/oembed\/1.0\/embed?url=https%3A%2F%2Fstatescoop.com%2Fnist-security-threats-ai-state-local%2F&format=xml\">   <link rel=\"icon\" href=\"https:\/\/statescoop.com\/wp-content\/uploads\/sites\/6\/2023\/01\/cropped-ss_favicon.png?w=32\" sizes=\"32x32\">\n<link rel=\"icon\" href=\"https:\/\/statescoop.com\/wp-content\/uploads\/sites\/6\/2023\/01\/cropped-ss_favicon.png?w=192\" sizes=\"192x192\">\n<link rel=\"apple-touch-icon\" href=\"https:\/\/statescoop.com\/wp-content\/uploads\/sites\/6\/2023\/01\/cropped-ss_favicon.png?w=180\">\n<meta name=\"msapplication-TileImage\" content=\"https:\/\/statescoop.com\/wp-content\/uploads\/sites\/6\/2023\/01\/cropped-ss_favicon.png?w=270\"> <\/head><body class=\"post-template-default single single-post postid-61230 single-format-standard\" id=\"readabilityBody\"> <a href=\"https:\/\/statescoop.com\/nist-security-threats-ai-state-local\/#main\" class=\"skip-to-content-link visually-hidden-focusable\">Skip to main content<\/a> <\/p>\n<div class=\"ad ad--top ad--top-desktop\">\n<div class=\"ad__inner\"> <span class=\"screen-reader-text\">Advertisement<\/span> <\/div>\n<\/div>\n<p> <main id=\"main\" role=\"main\" tabindex=\"-1\"> <\/p>\n<div class=\"ad ad--top ad--top-mobile\">\n<div class=\"ad__inner\"> <span class=\"screen-reader-text\">Advertisement<\/span> <\/div>\n<\/div>\n<section id=\"stickybar\" class=\"stickybar stickybar--newsletter js-stickybar\" readability=\"0.82\"> <button class=\"stickybar__close js-stickybar-close\" aria-controls=\"stickybar\"> <svg class=\"icon icon--close\" width=\"21\" height=\"22\" viewBox=\"0 0 21 22\" fill=\"none\"><path d=\"m.822.518-.805.805L9.695 11 .017 20.678l.805.805 9.678-9.678 9.677 9.678.806-.805L11.305 11l9.678-9.677-.806-.805-9.677 9.677L.822.518Z\" fill=\"currentColor\" \/><\/svg> <span class=\"visually-hidden\">Close<\/span> <\/button> <\/section>\n<article class=\"single-article content\">\n<div class=\"single-article__container js-single-article-content\">\n<header class=\"single-article__header \" readability=\"24.8625\">\n<div class=\"single-article__header-content\" readability=\"31.128888888889\">\n<p> State and local governments are among the organizations threatened by various exploits against AI systems, according to a recent paper. <\/p>\n<\/p><\/div>\n<div class=\"single-article__cover-wrap\">\n<figure class=\"single-article__cover\"> <img data-recalc-dims=\"1\" fetchpriority=\"high\" width=\"640\" height=\"360\" src=\"https:\/\/i0.wp.com\/ddi.mohflo.net\/wp-content\/uploads\/2024\/01\/nist-researchers-warn-of-top-ai-security-threats.jpg?resize=640%2C360&ssl=1\" class=\"single-article__cover-image wp-post-image\" alt=\"digital head\" decoding=\"async\" srcset=\"https:\/\/ddi.mohflo.net\/wp-content\/uploads\/2024\/01\/nist-researchers-warn-of-top-ai-security-threats-2.jpg 1802w, https:\/\/ddi.mohflo.net\/wp-content\/uploads\/2024\/01\/nist-researchers-warn-of-top-ai-security-threats-2.jpg?resize=300,168 300w, https:\/\/ddi.mohflo.net\/wp-content\/uploads\/2024\/01\/nist-researchers-warn-of-top-ai-security-threats-2.jpg?resize=768,432 768w, https:\/\/ddi.mohflo.net\/wp-content\/uploads\/2024\/01\/nist-researchers-warn-of-top-ai-security-threats-2.jpg?resize=1024,576 1024w, https:\/\/ddi.mohflo.net\/wp-content\/uploads\/2024\/01\/nist-researchers-warn-of-top-ai-security-threats-2.jpg?resize=1536,864 1536w, https:\/\/ddi.mohflo.net\/wp-content\/uploads\/2024\/01\/nist-researchers-warn-of-top-ai-security-threats-2.jpg?resize=600,337 600w, https:\/\/ddi.mohflo.net\/wp-content\/uploads\/2024\/01\/nist-researchers-warn-of-top-ai-security-threats-2.jpg?resize=1200,675 1200w, https:\/\/ddi.mohflo.net\/wp-content\/uploads\/2024\/01\/nist-researchers-warn-of-top-ai-security-threats-2.jpg?resize=1498,843 1498w\" sizes=\"(max-width: 1200px) 100vw, 1200px\"><figcaption> (Getty Images) <\/figcaption><\/figure>\n<\/p><\/div>\n<\/header>\n<div class=\"single-article__content\">\n<div class=\"single-article__content-inner has-drop-cap\"> <html readability=\"73.009332536957\"><body readability=\"147.18734793187\"><\/p>\n<p>As <a href=\"http:\/\/statescoop.com\/state-government-generative-ai-uses\/\">dozens of states<\/a> race to establish standards for how their agencies use AI to increase efficiency and streamline public-facing services, researchers at the <a href=\"https:\/\/www.nist.gov\/\">National Institute of Standards and Technology<\/a> found that artificial intelligence systems, which rely on large amounts of data to perform tasks, can malfunction when exposed to untrustworthy data, according a report published last week.<\/p>\n<p>The <a href=\"https:\/\/csrc.nist.gov\/pubs\/ai\/100\/2\/e2023\/final\">report<\/a>, part of a broader effort by the institute to support the development of trustworthy AI, found that cyber criminals can deliberately confuse or \u201cpoison\u201d AI systems to make them malfunction by exposing them to bad data. And what\u2019s more, according to the study, there\u2019s no one-size-fits-all defense that developers or cybersecurity experts can implement to protect AI systems.<\/p>\n<p>\u201cData is incredibly important for machine learning,\u201d NIST computer scientist Apostol Vassilev, one of the publication\u2019s authors, told StateScoop. \u201c\u2018Garbage in, garbage out\u2019 is a well known kind of catchphrase in the trade.\u201d<\/p>\n<p>To perform tasks like autonomously driving vehicles or interacting with customers as online chatbots, AI is trained on vast quantities of data, which help the technology predict how best to respond in a variety of situations. Autonomous vehicles, for example, are trained on images of highways and streets with road signs, among other datasets. A chatbot might be exposed to records of online conversations.<\/p>\n<div class=\"ad ad--inline_1 \">\n<div class=\"ad__inner\"> <span class=\"screen-reader-text\">Advertisement<\/span> <\/div>\n<\/div>\n<p>Researchers warned that some AI training data \u2014 such as websites with inaccurate information or undesirable interactions with the public \u2014 may not be trustworthy and could cause AI systems to perform in an unintended manner. Chatbots, for example, might learn to respond with abusive or racist language when their guardrails get circumvented by carefully crafted malicious prompts. <\/p>\n<p>Joseph Thacker, a principal AI engineer and security researcher at <a href=\"https:\/\/appomni.com\/\">AppOmni<\/a>, security management software used by state and local governments, said it\u2019s important to consider the security protocols needed to safeguard against every potential attack \u2014 like the ones outlined in NIST\u2019s report.<\/p>\n<p>\u201cWe\u2019re gonna need everyone\u2019s help to secure it,\u201d Thacker told StateScoop. \u201cAnd I think people should be thinking that through.\u201d<\/p>\n<h3 class=\"wp-block-heading\" id=\"h-malicious-intent\">\u2018Malicious intent\u2019<\/h3>\n<p>The NIST report outlined four types of attacks on AI \u2014 poisoning, evasion, privacy and abuse \u2014 and classified them based on criteria such as the attacker\u2019s goals and objectives, capabilities and system knowledge.<\/p>\n<div class=\"ad ad--inline_1 \">\n<div class=\"ad__inner\"> <span class=\"screen-reader-text\">Advertisement<\/span> <\/div>\n<\/div>\n<p>Poisoning occurs when an AI system is trained on corrupted data, such as by slipping numerous instances of inappropriate language into conversation records so that a chatbot interprets those instances as a common enough occurrence to use in its own customer interactions.<\/p>\n<p>\u201cUsing a generative AI example, if you have a malicious intent and try to modify some of this input data that is fed into the model during training, where the model learns how to classify what is a cat, what is a dog and all these things, it can actually learn perturbations that could cause the model to misclassify, \u201d explained Apostol Vassilev, one of the NIST computer scientists who wrote the report.<\/p>\n<p>But Thacker, who specializes in application security, hacking and AI, argued that while data poisoning is possible, its window is limited to the tool\u2019s training phase and the other types of attacks \u2014 evasion, privacy and abuse in the form of prompt injections \u2014 are therefore more likely.<\/p>\n<p>\u201cIf you can evade the filter, then that is an attack on the system, because you\u2019re bypassing the set protection,\u201d Thacker said of prompt injections, when bad actors trick the system into voluntarily offering someone else\u2019s data.<\/p>\n<p>Thacker said prompt injection attacks aim to force a chatbot to provide sensitive training data it\u2019s programmed to withhold.<\/p>\n<div class=\"ad ad--inline_1 \">\n<div class=\"ad__inner\"> <span class=\"screen-reader-text\">Advertisement<\/span> <\/div>\n<\/div>\n<p>\u201cIf you\u2019re able to extract data directly out of the model that went into the training of it \u2014 and a lot of times it\u2019s trained on all the data on the internet, which will often contain a lot of people\u2019s private information,\u201d Thacker said. \u201c If you\u2019re able to get the large language model to then output that sensitive information, it violates the privacy of that person.\u201d<\/p>\n<h3 class=\"wp-block-heading\" id=\"h-so-what-can-be-done\">So what can be done?<\/h3>\n<p>Vassilev said a top challenge for state and local governments is incorporating large language models into their workflows securely. And while there are ways to mitigate attacks against AI, he cautioned agencies not to fall into a false sense of security, because there\u2019s no foolproof method of protecting AI from misdirection.<\/p>\n<p>\u201cYou can\u2019t just say \u2018Okay, I got this model and apply this technique and I\u2019m done.\u2019 What you need to do is continue to monitor, assess and react when problems occur,\u201d said Vassilev, who also acknowledged that researchers should also develop better cybersecurity defenses. \u201cIn the meantime, you guys have to be alert and aware of all of these things. And monitor continuously.\u201d<\/p>\n<p>Thacker, who helps tech companies find these kinds of vulnerabilities in their software, insisted there are some common-sense ways to protect against AI security threats, including prohibiting access to sensitive data.<\/p>\n<div class=\"ad ad--inline_1 \">\n<div class=\"ad__inner\"> <span class=\"screen-reader-text\">Advertisement<\/span> <\/div>\n<\/div>\n<p>\u201cDon\u2019t connect systems that have access to sensitive data, like Social Security numbers or other personal information,\u201d Thacker said. \u201cIf a government agency wants to enable its employees to work more efficiently through the use of AI, like ChatGPT or a similar service, don\u2019t put in [training] data that\u2019s sensitive. And don\u2019t hook that up to a system which allows access to that data either.\u201d<\/p>\n<p>But Thacker also sounded a note of optimism, predicting that AI\u2019s security features will become more common, similar to the ubiquity of two-factor authentication.<\/p>\n<p>\u201cA lot of people don\u2019t realize everything that\u2019s beneath the waters when they kind of are using a website or using a [software-as-a-service] application\u201d he said. \u201cI think that AI security is going to be integrated through the tech stack of your traditional security, and then your cloud security and then your SaaS security.\u201d<\/p>\n<p><\/body> <\/p>\n<footer class=\"single-article__footer\" readability=\"3.5555555555556\">\n<div class=\"author-card\" readability=\"13\">\n<div class=\"author-card__avatar\">\n<figure class=\"author-card__image-wrap\"> <img data-recalc-dims=\"1\" decoding=\"async\" class=\"author-card__image\" src=\"https:\/\/i0.wp.com\/ddi.mohflo.net\/wp-content\/uploads\/2024\/01\/nist-researchers-warn-of-top-ai-security-threats-1.jpg?w=640&ssl=1\" alt=\"Sophia Fox-Sowell\"> <\/figure>\n<\/p><\/div>\n<p><h4 class=\"author-card__name\">Written by Sophia Fox-Sowell<\/h4>\n<p> Sophia Fox-Sowell reports on artificial intelligence, cybersecurity and government regulation for StateScoop. She was previously a multimedia producer for CNET, where her coverage focused on private sector innovation in food production, climate change and space through podcasts and video content. She earned her bachelor\u2019s in anthropology at Wagner College and master\u2019s in media innovation from Northeastern University. <\/p>\n<\/p><\/div>\n<div class=\"single-article__tags-container\">\n<h4 class=\"single-article__tags-title\">In This Story<\/h4>\n<\/p><\/div>\n<\/footer>\n<p> <\/html><\/div>\n<\/p><\/div>\n<\/p><\/div>\n<div class=\"single-article__ads js-single-article-sidebar\">\n<div class=\"ad ad--sidebar js-single-article-sidebar-5 ad--rightrail_1 \">\n<div class=\"ad__inner\"> <span class=\"screen-reader-text\">Advertisement<\/span> <\/div>\n<\/div>\n<div class=\"ad ad--sidebar js-single-article-sidebar-4 ad--rightrail_2 \">\n<div class=\"ad__inner\"> <span class=\"screen-reader-text\">Advertisement<\/span> <\/div>\n<\/div>\n<div class=\"ad ad--sidebar js-single-article-sidebar-3 ad--rightrail_3 \">\n<div class=\"ad__inner\"> <span class=\"screen-reader-text\">Advertisement<\/span> <\/div>\n<\/div><\/div>\n<\/article>\n<div class=\"popular-stories popular-stories--single-post\">\n<div class=\"popular-stories__container\">\n<h2 class=\"popular-stories__title\"> More Scoops <\/h2>\n<p>  <\/div>\n<p>\n<\/div>\n<p> <\/p>\n<section class=\"latest-podcasts\">\n<h2 class=\"latest-podcasts__title\"> Latest Podcasts\t<\/h2>\n<\/section>\n<div class=\"top-categories\">\n<div class=\"top-categories__container\">\n<h3 class=\"top-categories__category-title\">Modernization<\/h3>\n<\/p><\/div>\n<div class=\"top-categories__container\">\n<h3 class=\"top-categories__category-title\">Cybersecurity<\/h3>\n<\/p><\/div>\n<\/p><\/div>\n<p> <\/main> <\/p>\n<div class=\"ad ad--bottom \">\n<div class=\"ad__inner\"> <span class=\"screen-reader-text\">Advertisement<\/span> <\/div>\n<\/div>\n<div id=\"interstitial\" class=\"welcome__container\"> <button id=\"close-modal-1\" class=\"welcome__clickable_area\"><\/button> <\/p>\n<div class=\"welcome__ad_wrapper\">\n<p> <button id=\"close-modal-3\" class=\"welcome__continue-button\">Continue to StateScoop<\/button> <\/p>\n<\/p><\/div>\n<\/p><\/div>\n<p>   <\/body> <a href=\"https:\/\/statescoop.com\/nist-security-threats-ai-state-local\/\">Source<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"<p>NIST researchers warn of top AI security threats | StateScoop<\/p>\n","protected":false},"author":11,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"_jetpack_memberships_contains_paid_content":false,"footnotes":""},"categories":[235,78],"tags":[236,86],"class_list":["post-2354","post","type-post","status-publish","format-standard","hentry","category-ai","category-cybersecurity","tag-ai","tag-cybersecurity"],"featured_image_urls":{"full":"","thumbnail":"","medium":"","medium_large":"","large":"","1536x1536":"","2048x2048":"","chromenews-featured":"","chromenews-large":"","chromenews-medium":""},"author_info":{"display_name":"Cyber Scoop","author_link":"https:\/\/ddi.mohflo.net\/index.php\/author\/cyberscoop\/"},"category_info":"<a href=\"https:\/\/ddi.mohflo.net\/index.php\/category\/ai\/\" rel=\"category tag\">AI<\/a> <a href=\"https:\/\/ddi.mohflo.net\/index.php\/category\/cybersecurity\/\" rel=\"category tag\">Cybersecurity<\/a>","tag_info":"Cybersecurity","comment_count":"0","jetpack_featured_media_url":"","jetpack_sharing_enabled":true,"_links":{"self":[{"href":"https:\/\/ddi.mohflo.net\/index.php\/wp-json\/wp\/v2\/posts\/2354","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/ddi.mohflo.net\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/ddi.mohflo.net\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/ddi.mohflo.net\/index.php\/wp-json\/wp\/v2\/users\/11"}],"replies":[{"embeddable":true,"href":"https:\/\/ddi.mohflo.net\/index.php\/wp-json\/wp\/v2\/comments?post=2354"}],"version-history":[{"count":0,"href":"https:\/\/ddi.mohflo.net\/index.php\/wp-json\/wp\/v2\/posts\/2354\/revisions"}],"wp:attachment":[{"href":"https:\/\/ddi.mohflo.net\/index.php\/wp-json\/wp\/v2\/media?parent=2354"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/ddi.mohflo.net\/index.php\/wp-json\/wp\/v2\/categories?post=2354"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/ddi.mohflo.net\/index.php\/wp-json\/wp\/v2\/tags?post=2354"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}

{"id":2354,"date":"2024-01-11T20:52:31","date_gmt":"2024-01-11T20:52:31","guid":{"rendered":"https:\/\/cyberscoop.com\/?p=78765"},"modified":"2024-01-11T20:52:31","modified_gmt":"2024-01-11T20:52:31","slug":"nist-researchers-warn-of-top-ai-security-threats","status":"publish","type":"post","link":"https:\/\/ddi.mohflo.net\/index.php\/2024\/01\/11\/nist-researchers-warn-of-top-ai-security-threats\/","title":{"rendered":"NIST researchers warn of top AI security threats"},"content":{"rendered":"