diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..a971dfb32 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[report] +include = security_monkey/*.py + diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..1750cd364 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.git +secmonkey.env +boto.cfg +.travis.yml +#docs +supervisor +config-default.py +generate-docs.py diff --git a/.gitignore b/.gitignore index 18a7937b3..2de573b06 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,8 @@ devlog/ venv/ .idea/ +boto.cfg +secmonkey.env +*.crt +*.key + diff --git a/.travis.yml b/.travis.yml index 450584f32..5b5e3db54 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -sudo: false +sudo: required language: python @@ -20,18 +20,28 @@ env: install: +before_install: + # - sudo apt-get -qq update + # - sudo apt-get install -y libxml2-dev libxmlsec1-dev + before_script: - psql -c "CREATE DATABASE securitymonkeydb;" -U postgres - psql -c "CREATE ROLE securitymonkeyuser LOGIN PASSWORD 'securitymonkeypass';" -U postgres - psql -c "CREATE SCHEMA securitymonkeydb GRANT Usage, Create ON SCHEMA securitymonkeydb TO securitymonkeyuser;" -U postgres - psql -c "set timezone TO 'GMT';" -U postgres - python setup.py develop + - pip install .[tests] + - pip install coveralls - python manage.py db upgrade script: - sh env_tests/test_dart.sh - - py.test security_monkey/tests || exit 1 + - coverage run -m py.test security_monkey/tests || exit 1 + +after_success: + - coveralls notifications: email: - mgrima@netflix.com + - mgrima@netflix.com + - pkelley@netflix.com diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..2959018a7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,46 @@ + +# Copyright 2014 Netflix, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:14.04 +MAINTAINER Netflix Open Source Development + +ENV SECURITY_MONKEY_VERSION=v0.8.0 \ + SECURITY_MONKEY_SETTINGS=/usr/local/src/security_monkey/env-config/config-docker.py + +RUN apt-get update &&\ + apt-get -y -q install python-software-properties software-properties-common postgresql-9.3 postgresql-client-9.3 postgresql-contrib-9.3 curl &&\ + apt-get install -y python-pip python-dev python-psycopg2 libffi-dev libpq-dev libyaml-dev libxml2-dev libxmlsec1-dev git sudo swig &&\ + rm -rf /var/lib/apt/lists/* + +RUN pip install setuptools --upgrade + +RUN cd /usr/local/src &&\ +# git clone --branch $SECURITY_MONKEY_VERSION https://github.com/Netflix/security_monkey.git + /bin/mkdir -p security_monkey +ADD . /usr/local/src/security_monkey + +RUN cd /usr/local/src/security_monkey &&\ + python setup.py install &&\ + /bin/mkdir -p /var/log/security_monkey/ + +RUN chmod +x /usr/local/src/security_monkey/docker/*.sh &&\ + mkdir -pv /var/log/security_monkey &&\ + /usr/bin/touch /var/log/security_monkey/securitymonkey.log + # ln -s /dev/stdout /var/log/security_monkey/securitymonkey.log + +WORKDIR /usr/local/src/security_monkey +EXPOSE 5000 + +ENTRYPOINT ["/usr/local/src/security_monkey/docker/api-start.sh"] diff --git a/README.rst b/README.rst index 0f070eea2..33a72ce09 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,28 @@ .. image:: https://badge.waffle.io/Netflix/security_monkey.png?label=ready&title=Ready - :target: https://waffle.io/Netflix/security_monkey - :alt: 'Stories in Ready' + :target: https://waffle.io/Netflix/security_monkey + :alt: 'Stories in Ready' + +.. image:: https://badges.gitter.im/Join%20Chat.svg + :alt: Join the chat at https://gitter.im/Netflix/security_monkey + :target: https://gitter.im/Netflix/security_monkey?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge + +**develop branch**: + +.. image:: https://travis-ci.org/Netflix/security_monkey.svg?branch=develop + :target: https://travis-ci.org/Netflix/security_monkey + +.. image:: https://coveralls.io/repos/github/Netflix/security_monkey/badge.svg?branch=develop + :target: https://coveralls.io/github/Netflix/security_monkey + +**master branch**: + +.. image:: https://travis-ci.org/Netflix/security_monkey.svg?branch=master + :target: https://travis-ci.org/Netflix/security_monkey + +.. image:: https://coveralls.io/repos/github/Netflix/security_monkey/badge.svg?branch=master + :target: https://coveralls.io/github/Netflix/security_monkey + + *************** Security Monkey *************** @@ -16,9 +38,3 @@ Project resources - `Documentation `_ - `Source code `_ - `Issue tracker `_ - - - -.. image:: https://badges.gitter.im/Join%20Chat.svg - :alt: Join the chat at https://gitter.im/Netflix/security_monkey - :target: https://gitter.im/Netflix/security_monkey?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge \ No newline at end of file diff --git a/dart/lib/component/account_view_component/account_view_component.dart b/dart/lib/component/account_view_component/account_view_component.dart index d556b6c97..0ffc9ea42 100644 --- a/dart/lib/component/account_view_component/account_view_component.dart +++ b/dart/lib/component/account_view_component/account_view_component.dart @@ -12,22 +12,33 @@ class AccountViewComponent implements ScopeAware { Account account; bool create = false; bool _as_loaded = false; + bool _cfg_loaded = false; bool _is_error = false; String err_message = ""; + AccountConfig config; ObjectStore store; AccountViewComponent(this.routeProvider, this.router, this.store) { this.store = store; // If the URL has an ID, then let's view/edit if (routeProvider.parameters.containsKey("accountid")) { - store.one(Account, routeProvider.parameters['accountid']).then((Account account) { + store.one(Account, routeProvider.parameters['accountid']).then((account) { this.account = account; - _as_loaded = true; + this._as_loaded = true; + }); + store.one(AccountConfig, "all").then((account_config) { + this.config = account_config; + _cfg_loaded = true; }); create = false; } else { // If the URL does not have an ID, then let's create - account = new Account(); + this.account = new Account(); + store.one(AccountConfig, "all").then((account_config) { + this.config = account_config; + _cfg_loaded = true; + }); + _as_loaded = true; create = true; } } @@ -36,7 +47,7 @@ class AccountViewComponent implements ScopeAware { scope.on("globalAlert").listen(this._showMessage); } - get isLoaded => create || _as_loaded; + get isLoaded => _as_loaded && _cfg_loaded; get isError => _is_error; void _showMessage(ScopeEvent event) { @@ -53,7 +64,9 @@ class AccountViewComponent implements ScopeAware { }); }); } else { - this.store.update(this.account); + this.store.update(this.account).then((_) { + window.location.reload(); + }); } } diff --git a/dart/lib/component/account_view_component/account_view_component.html b/dart/lib/component/account_view_component/account_view_component.html index c46615fc7..f876ecc7a 100644 --- a/dart/lib/component/account_view_component/account_view_component.html +++ b/dart/lib/component/account_view_component/account_view_component.html @@ -26,26 +26,29 @@

Create Account

-
- +
- +
-
- +
+
- + +
@@ -90,6 +93,7 @@

Due to an open + +
+
+
+ +

Accounts

+
+
+ + + + + + + + + + + + + + + +
+ Select All + Clear All + + Account + + + Score + +
+ +
{{account.name}}{{account.total_score}}
+
+

Loading . . .

+
+
+
+
+ +
+
+
+

Technology Issue Scores

+
+
+
+

Loading . . .

+
+ + + + + + + + + + + + + +
+ Technology + + + Score + +
{{tech.name}}{{tech.score}}
+
+
+
+
+ +
+
+

High Score Items

+
+
+
+

Loading . . .

+
+ + + + + + + + + + + + + + + + + +
AccountTechnologyNameScore
{{highScoreItem.account}}{{highScoreItem.technology}}{{highScoreItem.name}}{{highScoreItem.totalScore()}}
+
+
+ +
+
+

Aging Issues

+
+
+
+

Loading . . .

+
+ + + + + + + + + + + + + + + + + + + + + +
AccountTechnologyItem NameIssueScoreIssue Date
{{agingIssue.item.account}}{{agingIssue.item.technology}}{{agingIssue.item.name}}{{agingIssue.issue}}{{agingIssue.score}}{{agingIssue.justified_date | date:'short'}} +
+
+
+
+

diff --git a/dart/lib/component/item_table_component/item_table_component.dart b/dart/lib/component/item_table_component/item_table_component.dart index c7ce2d31d..1bf1e1c2f 100644 --- a/dart/lib/component/item_table_component/item_table_component.dart +++ b/dart/lib/component/item_table_component/item_table_component.dart @@ -31,6 +31,8 @@ class ItemTableComponent extends PaginatedTable implements DetachAware { 'arns': '', 'active': null, 'searchconfig': null, + 'min_score': null, + 'min_unjustified_score': null, 'page': '1', 'count': '25' }; diff --git a/dart/lib/component/itemdetails/itemdetails.html b/dart/lib/component/itemdetails/itemdetails.html index 0c0e9bed0..15d5178c2 100644 --- a/dart/lib/component/itemdetails/itemdetails.html +++ b/dart/lib/component/itemdetails/itemdetails.html @@ -78,7 +78,14 @@
{{issue.issue}} {{issue.score}} - {{issue.notes}} + Related to: + + + {{issue.notes}}
@@ -112,7 +119,14 @@ {{issue.issue}} {{issue.justification}} {{issue.score}} - {{issue.notes}} + Related to: + + + {{issue.notes}}
+
+
Min Total Score
+ +
+ +
+
Min Unjustified Score
+ +
+
Status
+ + +
+
+
+ + + + diff --git a/dart/lib/component/settings/network_whitelist_component/network_whitelist_component.dart b/dart/lib/component/settings/network_whitelist_component/network_whitelist_component.dart new file mode 100644 index 000000000..25b5a156b --- /dev/null +++ b/dart/lib/component/settings/network_whitelist_component/network_whitelist_component.dart @@ -0,0 +1,49 @@ +part of security_monkey; + +@Component( + selector: 'whitelist-cmp', + templateUrl: 'packages/security_monkey/component/settings/network_whitelist_component/network_whitelist_component.html', + //cssUrl: const ['/css/bootstrap.min.css'] + useShadowDom: false +) +class NetworkWhitelistComponent extends PaginatedTable { + UsernameService us; + Router router; + List cidrs; + ObjectStore store; + + NetworkWhitelistComponent(this.router, this.store, this.us) { + cidrs = new List(); + list(); + } + + get signed_in => us.signed_in; + + void list() { + store.list(NetworkWhitelistEntry, params: { + "count": ipp_as_int, + "page": currentPage + }).then((cidrs) { + super.setPaginationData(cidrs.meta); + this.cidrs = cidrs; + super.is_loaded = true; + }); + } + + void createWhitelist() { + router.go('createwhitelist', {}); + } + + void deleteWhitelist(NetworkWhitelistEntry cidr){ + store.delete(cidr).then( (_) { + store.list(NetworkWhitelistEntry).then( (cidrs) { + this.cidrs = cidrs; + }); + }); + } + + String url_encode(input) => param_to_url(input); + + get isLoaded => super.is_loaded; + get isError => super.is_error; +} diff --git a/dart/lib/component/settings/network_whitelist_component/network_whitelist_component.html b/dart/lib/component/settings/network_whitelist_component/network_whitelist_component.html new file mode 100644 index 000000000..b11c4bf6e --- /dev/null +++ b/dart/lib/component/settings/network_whitelist_component/network_whitelist_component.html @@ -0,0 +1,53 @@ +
+
+
Network Whitelist {{ items_displayed() }} of {{ totalItems }}
+
+

Loading . . .

+
+ {{err_message}} +
+
+
+ + + + + + + + + + + + + + +
Network NameCIDRNotes
{{cidr.name}}{{cidr.name}}{{cidr.cidr}}{{cidr.notes}}
+
+
+
+
+ +
+
+
+ +
+
+
+ +
+
+
diff --git a/dart/lib/component/settings_component/settings_component.dart b/dart/lib/component/settings_component/settings_component.dart index cf7a7bd1f..c2ff1b5eb 100644 --- a/dart/lib/component/settings_component/settings_component.dart +++ b/dart/lib/component/settings_component/settings_component.dart @@ -10,35 +10,24 @@ class SettingsComponent extends PaginatedTable { UsernameService us; Router router; List accounts; - List cidrs; - List ignorelist; List auditorlist; ObjectStore store; UserSetting user_setting; - + SettingsComponent(this.router, this.store, this.us) { - cidrs = new List(); accounts = new List(); store.customQueryOne(UserSetting, new CustomRequestParams(method: "GET", url: "$API_HOST/settings", withCredentials: true)).then((user_setting) { this.user_setting = user_setting; list(); }); - store.list(NetworkWhitelistEntry).then( (cidrs) { - this.cidrs = cidrs; - }); - - store.list(IgnoreEntry).then( (ignoreItems) { - this.ignorelist = ignoreItems; - }); - store.list(AuditorSetting).then( (auditorItems) { this.auditorlist = auditorItems; }); } - + get signed_in => us.signed_in; - + void list() { store.list(Account, params: { "count": ipp_as_int, @@ -104,30 +93,6 @@ class SettingsComponent extends PaginatedTable { router.go('createaccount', {}); } - void createWhitelist() { - router.go('createwhitelist', {}); - } - - void deleteWhitelist(NetworkWhitelistEntry cidr){ - store.delete(cidr).then( (_) { - store.list(NetworkWhitelistEntry).then( (cidrs) { - this.cidrs = cidrs; - }); - }); - } - - void createIgnoreEntry() { - router.go('createignoreentry', {}); - } - - void deleteIgnoreList(ignoreitem) { - store.delete(ignoreitem).then( (_) { - store.list(IgnoreEntry).then( (ignoreitems) { - this.ignorelist = ignoreitems; - }); - }); - } - void disableAuditor(auditor) { auditor.disabled = true; store.update(auditor); @@ -137,7 +102,7 @@ class SettingsComponent extends PaginatedTable { auditor.disabled = false; store.update(auditor); } - + String url_encode(input) => param_to_url(input); get isLoaded => super.is_loaded; diff --git a/dart/lib/component/settings_component/settings_component.html b/dart/lib/component/settings_component/settings_component.html index 1a5890738..3aaeaf0e1 100644 --- a/dart/lib/component/settings_component/settings_component.html +++ b/dart/lib/component/settings_component/settings_component.html @@ -51,8 +51,8 @@
Daily Email
Active Third Party Name - S3 Name - Account Number + Type + Identifier Notes @@ -68,8 +68,8 @@
Daily Email
{{account.name}} {{account.name}} - {{account.s3_name}} - {{account.number}} + {{account.account_type}} + {{account.identifier}} {{account.notes}} @@ -107,59 +107,11 @@
Daily Email

-
-
-
Network Whitelist {{cidrs.length}} of {{cidrs.length}}
-
-

Loading . . .

-
-
- - - - - - - - - - - - - - -
Network NameCIDRNotes
{{cidr.name}}{{cidr.name}}{{cidr.cidr}}{{cidr.notes}}
-
-
-
+
-
-
-
Ignore List {{ignorelist.length}} of {{ignorelist.length}}
-
-

Loading . . .

-
-
- - - - - - - - - - - - - - -
PrefixNotesTechnology
{{ignoreitem.prefix}}{{ignoreitem.prefix}}{{ignoreitem.notes}}{{ignoreitem.technology}}
-
-
-
+
diff --git a/dart/lib/model/Account.dart b/dart/lib/model/Account.dart index 76c22afd0..945e6e49a 100644 --- a/dart/lib/model/Account.dart +++ b/dart/lib/model/Account.dart @@ -5,12 +5,12 @@ import 'dart:convert'; class Account { int id; String name; - String s3_name; - String number; + String identifier; String notes; - String role_name; bool _active; bool _third_party; + String account_type; + Map custom_field_values = new Map(); Account(); @@ -36,10 +36,15 @@ class Account { active = data['active']; third_party = data['third_party']; name = data['name']; - s3_name = data['s3_name']; - number = data['number']; + identifier = data['identifier']; notes = data['notes']; - role_name = data['role_name']; + account_type = data['account_type']; + + if (data.containsKey('custom_fields')) { + for (var field in data['custom_fields']) { + custom_field_values[field['name']] = field['value']; + } + } } String toJson() { @@ -48,10 +53,10 @@ class Account { "active": active, "third_party": third_party, "name": name, - "s3_name": s3_name, - "number": number, + "identifier": identifier, "notes": notes, - "role_name": role_name, + "account_type": account_type, + "custom_fields": custom_field_values }; return JSON.encode(objmap); } diff --git a/dart/lib/model/Issue.dart b/dart/lib/model/Issue.dart index 6c3e315ca..ce27c2140 100644 --- a/dart/lib/model/Issue.dart +++ b/dart/lib/model/Issue.dart @@ -1,6 +1,7 @@ library security_monkey.model_issue; import 'Item.dart'; +import 'ItemLink.dart'; import 'package:security_monkey/util/utils.dart' show localDateFromAPIDate; class Issue { @@ -17,6 +18,7 @@ class Issue { bool selected_for_justification; Item item; + List item_links = new List(); Issue.fromMap(Map data) { id = data['id']; @@ -36,6 +38,13 @@ class Issue { "item": data }); - + for (var item_link in data['item_links']) { + ItemLink linkObj = new ItemLink.fromMap(item_link); + item_links.add(linkObj); + } } + + get has_sub_item => this.item_links.length != 0; + + get get_links => '{{item_links.first.name}}'; } diff --git a/dart/lib/model/ItemLink.dart b/dart/lib/model/ItemLink.dart new file mode 100644 index 000000000..0eef79f76 --- /dev/null +++ b/dart/lib/model/ItemLink.dart @@ -0,0 +1,11 @@ +library security_monkey.item_link; + +class ItemLink { + int id; + String name; + + ItemLink.fromMap(Map data) { + id = data['id']; + name = data['name']; + } +} diff --git a/dart/lib/model/account_config.dart b/dart/lib/model/account_config.dart new file mode 100644 index 000000000..b21a22e4d --- /dev/null +++ b/dart/lib/model/account_config.dart @@ -0,0 +1,32 @@ +library security_monkey.account_config; + +import 'custom_field_config.dart'; + +class AccountConfig { + List account_types = new List(); + Map identifier_labels = new Map(); + Map identifier_tool_tips = new Map(); + Map> custom_fields = new Map>(); + + AccountConfig(); + + AccountConfig.fromMap(Map data) { + if (data.containsKey('custom_configs')) { + var acc_configs = data['custom_configs']; + acc_configs.forEach((k,v) => this._addToLists(k, v)); + } + } + + void _addToLists(account_type, values) { + this.account_types.add(account_type); + this.identifier_labels[account_type] = values['identifier_label']; + this.identifier_tool_tips[account_type] = values['identifier_tool_tip']; + + var list = new List(); + var field_config = values['custom_fields']; + for (var field in field_config) { + list.add(new CustomFieldConfig.fromMap(field)); + } + this.custom_fields[account_type] = list; + } +} diff --git a/dart/lib/model/custom_field_config.dart b/dart/lib/model/custom_field_config.dart new file mode 100644 index 000000000..e8575e9e1 --- /dev/null +++ b/dart/lib/model/custom_field_config.dart @@ -0,0 +1,17 @@ +library security_monkey.custom_field_config; + +class CustomFieldConfig { + String name; + String label; + bool editable; + String tool_tip; + bool password; + + CustomFieldConfig.fromMap(Map data) { + name = data['name']; + label = data['label']; + editable = data['editable']; + tool_tip = data['tool_tip']; + password = data['password']; + } +} diff --git a/dart/lib/model/hammock_config.dart b/dart/lib/model/hammock_config.dart index bafb4f44c..40ae37fba 100644 --- a/dart/lib/model/hammock_config.dart +++ b/dart/lib/model/hammock_config.dart @@ -13,18 +13,20 @@ import 'UserSetting.dart'; import 'ignore_entry.dart'; import 'User.dart'; import 'Role.dart'; +import 'account_config.dart'; @MirrorsUsed( targets: const[ Account, IgnoreEntry, Issue, AuditorSetting, Item, ItemComment, NetworkWhitelistEntry, - Revision, RevisionComment, UserSetting, User, Role], + Revision, RevisionComment, UserSetting, User, Role, + AccountConfig], override: '*') import 'dart:mirrors'; import 'package:security_monkey/util/constants.dart'; -final serializeAWSAccount = serializer("accounts", ["id", "active", "third_party", "name", "s3_name", "number", "notes", "role_name"]); +Resource serializeAccount(Account account) => resource("accounts", account.id, account.toJson()); final serializeIssue = serializer("issues", ["id", "score", "issue", "notes", "justified", "justified_user", "justification", "justified_date", "item_id"]); final serializeRevision = serializer("revisions", ["id", "item_id", "config", "active", "date_created", "diff_html"]); final serializeItem = serializer("items", ["id", "technology", "region", "account", "name"]); @@ -63,9 +65,15 @@ createHammockConfig(Injector inj) { }, "accounts": { "type": Account, - "serializer": serializeAWSAccount, + "serializer": serializeAccount, "deserializer": { - "query": deserializeAWSAccount + "query": deserializeAccount + } + }, + "account_config": { + "type": AccountConfig, + "deserializer": { + "query": deserializeAccountConfig } }, "issues": { @@ -146,7 +154,8 @@ serializer(type, attrs) { }; } -deserializeAWSAccount(r) => new Account.fromMap(r.content); +deserializeAccount(r) => new Account.fromMap(r.content); +deserializeAccountConfig(r) => new AccountConfig.fromMap(r.content); deserializeIssue(r) => new Issue.fromMap(r.content); deserializeRevision(r) => new Revision.fromMap(r.content); deserializeItem(r) => new Item.fromMap(r.content); diff --git a/dart/lib/routing/securitymonkey_router.dart b/dart/lib/routing/securitymonkey_router.dart index 12cc42ff0..d1025aad3 100644 --- a/dart/lib/routing/securitymonkey_router.dart +++ b/dart/lib/routing/securitymonkey_router.dart @@ -4,8 +4,18 @@ import 'package:angular/angular.dart'; void securityMonkeyRouteInitializer(Router router, RouteViewFactory views) { views.configure({ + 'dashboard': ngRoute( + path: '/dashboard', + mount: { + 'view': ngRoute( + path: '', + view: 'views/dashboard.html'), + 'view_default': ngRoute( + defaultRoute: true, + view: 'views/error.html') + }), 'items': ngRoute( - path: '/items/:regions/:technologies/:accounts/:names/:arns/:active/:searchconfig/:page/:count', + path: '/items/:regions/:technologies/:accounts/:names/:arns/:active/:searchconfig/:min_score/:min_unjustified_score/:page/:count', defaultRoute: true, mount: { 'view': ngRoute( diff --git a/dart/lib/security_monkey.dart b/dart/lib/security_monkey.dart index d6ea7f32b..d8738b294 100644 --- a/dart/lib/security_monkey.dart +++ b/dart/lib/security_monkey.dart @@ -34,6 +34,9 @@ import 'model/auditorsetting.dart'; import 'model/CloudTrail.dart'; import 'model/User.dart'; import 'model/Role.dart'; +import 'model/ItemLink.dart'; +import 'model/account_config.dart'; +import 'model/custom_field_config.dart'; // Routing import 'routing/securitymonkey_router.dart'; @@ -66,7 +69,10 @@ part 'component/whitelist_view_component/whitelist_view_component.dart'; part 'component/ignore_entry_component/ignore_entry_component.dart'; part 'component/username_component/username_component.dart'; part 'component/settings/auditor_settings_component/auditor_settings_component.dart'; +part 'component/dashboard_component/dashboard_component.dart'; part 'component/settings/user_role_component/user_role_component.dart'; +part 'component/settings/network_whitelist_component/network_whitelist_component.dart'; +part 'component/settings/ignore_list_component/ignore_list_component.dart'; class SecurityMonkeyModule extends Module { @@ -100,7 +106,10 @@ class SecurityMonkeyModule extends Module { bind(IgnoreEntryComponent); bind(UsernameComponent); bind(AuditorSettingsComponent); + bind(DashboardComponent); bind(UserRoleComponent); + bind(NetworkWhitelistComponent); + bind(IgnoreListComponent); // Services bind(JustificationService); diff --git a/dart/pubspec.yaml b/dart/pubspec.yaml index 057fe79ce..3987fcb17 100644 --- a/dart/pubspec.yaml +++ b/dart/pubspec.yaml @@ -1,6 +1,6 @@ name: security_monkey description: An AWS Policy Monitoring and Alerting Tool -version: 0.6.0 +version: 0.8.0 dependencies: angular: "^1.1.2+2" angular_ui: ">=0.6.8 <0.7.0" diff --git a/dart/web/js/sso.js b/dart/web/js/sso.js index 2e0965f79..7104ae7f9 100644 --- a/dart/web/js/sso.js +++ b/dart/web/js/sso.js @@ -13,13 +13,20 @@ $.urlParam = function(name){ var create_url = function(provider) { var next = $.urlParam("next"); var url = provider.authorizationEndpoint; - url += "?response_type="+provider.responseType; - url += "&client_id="+provider.clientId; - url += "&redirect_uri="+provider.redirectUri; - url += "&scope="+provider.scope.join(provider.scopeDelimiter); - url += "&state=clientId,"+provider.clientId+",redirectUri,"+provider.redirectUri+",return_to,"+next; - if (provider.hd) { - url += "&hd="+provider.hd; + + if (provider.name.toLowerCase() == "onelogin") { + url += "?return_to="+next; + } else { // google/ping + url += "?"; + url += "response_type="+provider.responseType; + url += "&client_id="+provider.clientId; + url += "&redirect_uri="+provider.redirectUri; + url += "&scope="+provider.scope.join(provider.scopeDelimiter); + url += "&state=clientId,"+provider.clientId+",redirectUri,"+provider.redirectUri+",return_to,"+next; + if (provider.hd) { + url += "&hd="+provider.hd; + } + } return url; }; diff --git a/dart/web/ui.html b/dart/web/ui.html index c09d2a4cd..e432fd296 100644 --- a/dart/web/ui.html +++ b/dart/web/ui.html @@ -42,25 +42,42 @@
    -
  • Search
    • +
  • Dashboard
    • +
  • Search
  • Reports
    • diff --git a/dart/web/views/dashboard.html b/dart/web/views/dashboard.html new file mode 100644 index 000000000..43caf36cb --- /dev/null +++ b/dart/web/views/dashboard.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/data/aws_accounts.json b/data/aws_accounts.json index 5983d648c..78e5c5d6c 100644 --- a/data/aws_accounts.json +++ b/data/aws_accounts.json @@ -22,6 +22,10 @@ "region": "ap-southeast-2", "account_id": "284668455005" }, + { + "region": "ca-central-1", + "account_id": "819402241893" + }, { "region": "eu-central-1", "account_id": "035351147821" @@ -38,6 +42,10 @@ "region": "us-east-1", "account_id": "086441151436" }, + { + "region": "us-east-2", + "account_id": "475085895292" + }, { "region": "us-west-1", "account_id": "388731089494" @@ -49,12 +57,16 @@ ] }, "ELB Logs": { - "url": "http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/enable-access-logs.html", + "url": "https://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/enable-access-logs.html", "accounts": [ { "region": "us-east-1", "account_id": "127311923021" }, + { + "region": "us-east-2", + "account_id": "033677994240" + }, { "region": "us-west-1", "account_id": "027434742980" @@ -106,12 +118,16 @@ ] }, "RedShift Logs": { - "url": "http://docs.aws.amazon.com/redshift/latest/mgmt/db-auditing.html", + "url": "https://docs.aws.amazon.com/redshift/latest/mgmt/db-auditing.html", "accounts": [ { "region": "us-east-1", "account_id": "193672423079" }, + { + "region": "us-east-2", + "account_id": "391106570357" + }, { "region": "us-west-2", "account_id": "902366379725" @@ -124,6 +140,10 @@ "region": "eu-west-1", "account_id": "210876761215" }, + { + "region": "ca-central-1", + "account_id": "907379612154" + }, { "region": "ap-northeast-1", "account_id": "404641285394" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..57d61f52a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,86 @@ +--- + +### +# +# Documentation: http://securitymonkey.readthedocs.io/en/latest/index.html +# http://securitymonkey.readthedocs.io/en/latest/docker.html +# +# shortcuts +# open https://$(docker-machine active | xargs docker-machine ip) +# +### + + +version: '2' +services: + postgres: + container_name: secmonkey-db + image: postgres:9 + # volumes: + # - ./postgres-data/:/var/lib/postgresql/data + + api: + container_name: secmonkey-api + image: secmonkey:latest + volumes_from: + - init + depends_on: + - postgres + env_file: secmonkey.env + entrypoint: ["/usr/local/src/security_monkey/docker/api-start.sh"] + + scheduler: + container_name: secmonkey-scheduler + image: secmonkey:latest + volumes_from: + - init + depends_on: + - api + env_file: secmonkey.env + entrypoint: ["/usr/local/src/security_monkey/docker/scheduler-start.sh"] + + nginx: + container_name: secmonkey-nginx + build: + context: ./ + dockerfile: ./docker/nginx/Dockerfile + image: secmonkey-nginx:latest + working_dir: /etc/nginx + volumes: + - ./docker/nginx/server.crt:/etc/nginx/ssl/server.crt + - ./docker/nginx/server.key:/etc/nginx/ssl/server.key + - ./docker/nginx/securitymonkey.conf:/etc/nginx/conf.d/securitymonkey.conf + - ./docker/nginx/start-nginx.sh:/usr/local/src/security_monkey/docker/nginx/start-nginx.sh + depends_on: + - api + ports: + - 80:80 + - 443:443 + links: + - api:smapi + +# volumes: +# - postgres-data: {} + +### ### ### + ### ### ### + + init: + container_name: init + build: . + image: secmonkey:latest + working_dir: /usr/local/src/security_monkey + volumes: + - ./data/aws_accounts.json:/usr/local/src/security_monkey/data/aws_accounts.json + - ./docker:/usr/local/src/security_monkey/docker/ + - ./env-config/config-docker.py:/usr/local/src/security_monkey/env-config/config-docker.py + depends_on: + - postgres + env_file: secmonkey.env + # environment: + # - AWS_ACCESS_KEY_ID= + # - AWS_SECRET_ACCESS_KEY= + # - SECURITY_MONKEY_POSTGRES_HOST= + entrypoint: # /usr/local/src/security_monkey/docker/api-init.sh + - sleep + - 8h diff --git a/docker/README.rst b/docker/README.rst new file mode 100644 index 000000000..04d86b4e2 --- /dev/null +++ b/docker/README.rst @@ -0,0 +1,9 @@ +************************ +Docker local development +************************ + +Project resources +================= + +- `Docker documentation `_ +- `Development documentation `_ diff --git a/docker/api-init.sh b/docker/api-init.sh new file mode 100755 index 000000000..74bb8182f --- /dev/null +++ b/docker/api-init.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +sudo -u ${SECURITY_MONKEY_POSTGRES_USER:-postgres} psql\ + -h ${SECURITY_MONKEY_POSTGRES_HOST:-postgres} -p ${SECURITY_MONKEY_POSTGRES_PORT:-5432}\ + --command "ALTER USER ${SECURITY_MONKEY_POSTGRES_USER:-postgres} with PASSWORD '${SECURITY_MONKEY_POSTGRES_PASSWORD:-securitymonkeypassword}';" + +sudo -u ${SECURITY_MONKEY_POSTGRES_USER:-postgres} createdb\ + -h ${SECURITY_MONKEY_POSTGRES_HOST:-postgres} -p ${SECURITY_MONKEY_POSTGRES_PORT:-5432}\ + -O ${SECURITY_MONKEY_POSTGRES_USER:-postgres} ${SECURITY_MONKEY_POSTGRES_DATABASE:-secmonkey} + +mkdir -p /var/log/security_monkey/ +touch "/var/log/security_monkey/security_monkey-deploy.log" + +cd /usr/local/src/security_monkey +python manage.py db upgrade diff --git a/docker/api-start.sh b/docker/api-start.sh new file mode 100755 index 000000000..d02689d4a --- /dev/null +++ b/docker/api-start.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +cd /usr/local/src/security_monkey +python manage.py run_api_server -b 0.0.0.0:${SECURITY_MONKEY_API_PORT:-5000} diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile new file mode 100644 index 000000000..7fb90f3b6 --- /dev/null +++ b/docker/nginx/Dockerfile @@ -0,0 +1,50 @@ +# Copyright 2014 Netflix, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM nginx:1.11.4 +MAINTAINER Netflix Open Source Development + +ENV SECURITY_MONKEY_VERSION=v0.8.0 +RUN apt-get update &&\ + apt-get install -y curl git sudo apt-transport-https &&\ + curl https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - &&\ + curl https://storage.googleapis.com/download.dartlang.org/linux/debian/dart_stable.list > /etc/apt/sources.list.d/dart_stable.list && \ + apt-get update &&\ + apt-get install -y -q dart &&\ + rm -rf /var/lib/apt/lists/* + +RUN cd /usr/local/src &&\ +# git clone -b $SECURITY_MONKEY_VERSION https://github.com/Netflix/security_monkey.git + mkdir -p security_monkey +ADD . /usr/local/src/security_monkey + +RUN cd /usr/local/src/security_monkey/dart &&\ + /usr/lib/dart/bin/pub get &&\ + /usr/lib/dart/bin/pub build &&\ + /bin/mkdir -p /usr/local/src/security_monkey/security_monkey/static/ &&\ + /bin/cp -R /usr/local/src/security_monkey/dart/build/web/* /usr/local/src/security_monkey/security_monkey/static/ + +RUN /bin/rm /etc/nginx/conf.d/default.conf &&\ + /bin/mkdir -p /var/log/security_monkey/ /etc/nginx/ssl/ &&\ + ln -s /dev/stdout /var/log/security_monkey/security_monkey.access.log &&\ + ln -s /dev/stderr /var/log/security_monkey/security_monkey.error.log + +WORKDIR /etc/nginx +EXPOSE 443 + +ADD docker/nginx/securitymonkey.conf /etc/nginx/conf.d/securitymonkey.conf +COPY docker/nginx/nginx.conf /etc/nginx/nginx.conf +# ADD docker/nginx/server.crt docker/nginx/server.key /etc/nginx/ssl/ + +ENTRYPOINT ["/usr/local/src/security_monkey/docker/nginx/start-nginx.sh"] diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf new file mode 100644 index 000000000..7e0e29ff1 --- /dev/null +++ b/docker/nginx/nginx.conf @@ -0,0 +1,33 @@ + +user nginx; +worker_processes 1; +daemon off; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; +} diff --git a/docker/nginx/securitymonkey.conf b/docker/nginx/securitymonkey.conf new file mode 100644 index 000000000..75489d142 --- /dev/null +++ b/docker/nginx/securitymonkey.conf @@ -0,0 +1,37 @@ +add_header X-Content-Type-Options "nosniff"; +add_header X-XSS-Protection "1; mode=block"; +add_header X-Frame-Options "SAMEORIGIN"; +add_header Strict-Transport-Security "max-age=631138519"; +add_header Content-Security-Policy "default-src 'self'; font-src 'self' https://fonts.gstatic.com; script-src 'self' https://ajax.googleapis.com; style-src 'self' https://fonts.googleapis.com;"; + +server { + listen 0.0.0.0:80; + listen 0.0.0.0:443 ssl; + ssl_certificate /etc/nginx/ssl/server.crt; + ssl_certificate_key /etc/nginx/ssl/server.key; + access_log /var/log/security_monkey/security_monkey.access.log; + error_log /var/log/security_monkey/security_monkey.error.log; + + location ~* ^/(reset|confirm|healthcheck|register|login|logout|api) { + proxy_read_timeout 120; + proxy_pass http://smapi:5000; + proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; + proxy_redirect off; + proxy_buffering off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /static { + rewrite ^/static/(.*)$ /$1 break; + root /usr/local/src/security_monkey/security_monkey/static; + index ui.html; + } + + location / { + root /usr/local/src/security_monkey/security_monkey/static; + index ui.html; + } + +} diff --git a/docker/nginx/start-nginx.sh b/docker/nginx/start-nginx.sh new file mode 100755 index 000000000..efeed4d59 --- /dev/null +++ b/docker/nginx/start-nginx.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +SECURITY_MONKEY_SSL_CERT=${SECURITY_MONKEY_SSL_CERT:-/etc/nginx/ssl/server.crt} +SECURITY_MONKEY_SSL_KEY=${SECURITY_MONKEY_SSL_KEY:-/etc/nginx/ssl/server.key} + +if [ ! -f "$SECURITY_MONKEY_SSL_CERT" ] || [ ! -f "$SECURITY_MONKEY_SSL_KEY" ]; then + # Fail if SSL is unavailable + echo "$(date) Error: Missing files required for SSL" + # exit 1 + sed -i.bak 's@.*ssl@# &@' /etc/nginx/conf.d/securitymonkey.conf &&\ + echo "$(date) Warn: Disabled ssl in securitymonkey.conf" +fi + +exec nginx diff --git a/docker/scheduler-start.sh b/docker/scheduler-start.sh new file mode 100755 index 000000000..0f70228f1 --- /dev/null +++ b/docker/scheduler-start.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +mkdir -p /var/log/security_monkey +touch /var/log/security_monkey/security_monkey-deploy.log + +cd /usr/local/src/security_monkey +python manage.py start_scheduler diff --git a/docs/api/index.rst b/docs/api/index.rst index a23337b41..c16f1e737 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -28,7 +28,7 @@ included, and users are free to add their own rules. >>> import security_monkey >>> security_monkey.__version__ - u'0.6.0' + u'0.8.0' Class and method level definitions and documentation diff --git a/docs/authors.rst b/docs/authors.rst index 3f034559a..dcb4ccd5d 100644 --- a/docs/authors.rst +++ b/docs/authors.rst @@ -2,7 +2,7 @@ Authors ******* -securitymonkey 0.6.0 is copyright 2014,2015,2016 Netflix. inc. +securitymonkey 0.8.0 is copyright 2014,2015,2016 Netflix. inc. .. include:: ../AUTHORS diff --git a/docs/changelog.rst b/docs/changelog.rst index 3c5c8edf1..6210b3e26 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,84 @@ Changelog ********* +v0.8.0 (2016-12-02-delayed->2017-01-13) +======================================= +- PR #425 - @crruthe - Fixed a few report hyperlinks. +- PR #428 - @nagwww - Documentation fix. Renamed `module: security_monkey.auditors.elb` to `module: security_monkey.auditors.elasticsearch_service` +- PR #424 - @mikegrima - OS X Install doc updates for El Capitan and higher. +- PR #426 - @mikegrima - Added "route53domains:getdomaindetail" to permissions doc. +- PR #427 - @mikegrima - Fix for ARN parsing of cloudfront ARNs. +- PR #431 - @mikegrima - Removed s3 ARN check for ElasticSearch Service. +- PR #448 - @zollman - Fix exception logging in store_exception. +- PR #444 - @zollman - Adds exception logging listener for appscheduler. +- PR #454 - @mikegrima - Updated S3 Permissions to reflect latest changes to cloudaux. +- PR #455 - @zollman - Add Dashboard. +- PR #456 - @zollman - Increase issue note size. +- PR #420 - @crruthe - Added support for SSO OneLogin. +- PR #432 - @robertoriv - Add pagination for whitelist and ignore list. +- PR #438 - @AngeloCiffa - Pin moto==0.4.25. (TODO: Bump Jinja2 version.) +- PR #433 - @jnbnyc - Added Docker/Docker Compose support for local dev. +- PR #408 - @zollman - Add support for custom account metadata. (An important step that will allow us to support multiple cloud providers in the future.) +- PR #439 - @monkeysecurity - Replace botor lib with Netflix CloudAux. +- PR #441 - @monkeysecurity - Auditor ChangeItems now receive ARN. +- PR #446 - @zollman - Fix item 'first_seen' query . +- PR #447 - @zollman - Refactor rdsdbcluster array params. +- PR #445 - @zollman - Make misfire grace time and reporter start time configurable. +- PR #451 - @monkeysecurity - Add coverage with Coveralls.io. +- PR #452 - @monkeysecurity - Refactor & add tests for the PolicyDiff module. +- PR #449 - @monkeysecurity - Refactoring s3 watcher to use Netflix CloudAux. +- PR #453 - @monkeysecurity - Fixing two policy diff cases. +- PR #442 - @monkeysecurity - Adding index to region. Dropping unused item.cloud. +- PR #450 - @monkeysecurity - Moved test & onelogin requirements to the setup.py extras_require section. +- PR #407 - @zollman - Link together issues by enabling auditor dependencies. +- PR #419 - @monkeysecurity - Auditor will now fix any issues that are not attached to an AuditorSetting. +- PR NONE - @monkeysecurity - Item View no longer returns revision configuration bodies. Should improve UI for items with many revisions. +- PR NONE - @monkeysecurity - Fixing bug where SSO arguments weren't passed along for branded sso. (Where the name is not google or ping or onelogin) +- PR #476 - @markofu - Update aws_accounts.json to add Canada and Ohio regions. +- PR NONE - @monkeysecurity - Fixing `manage.py::amazon_accounts()` to use new AccountType and adding `delete_unjustified_issues()`. +- PR #480 - @monkeysecurity - Making Gunicorn an optional import to help support dev on Windows. +- PR #481 - @monkeysecurity - Fixing a couple dart warnings. +- PR #482 - @monkeysecurity - Replacing `Flask-Security` with `Flask-Security-Fork`. +- PR #483 - @monkeysecurity - issue #477 - Fixes IAM User Auditor login_profile check. +- PR #484 - @monkeysecurity - Bumping Jinja2 to `>=2.8.1` +- PR #485 - @robertoriv - New IAM Role Auditor feature - Check for unknown cross account assumerole. +- PR #487 - @hyperbolist - issue #486 - Upgrade setuptools in Dockerfile. +- PR #489 - @monkeysecurity - issue #251 - Fix IAM SSL Auditor regression. Issue should be raised if we cannot obtain cert issuer. +- PR #490 - @monkeysecurity - issue #421 - Adding ephemeral field to RDS DB issue. +- PR #491 - @monkeysecurity - Adding new RDS DB Cluster ephemeral field. +- PR #492 - @monkeysecurity - issue #466 - Updating S3 Auditor to use the ARN class. +- PR NONE - @monkeysecurity - Fixing typo in dart files. +- PR #495 - @monkeysecurity - issue #494 - Refactoring to work with the new Flask-WTF. +- PR #493 - @monkeysecurity - Windows 10 Development instructions. +- PR NONE - @monkeysecurity - issue #496 - Bumping CloudAux to >=1.0.7 to fix IAM User UploadDate field JSON serialization error. + +Important Notes: + +- New permissions required: + - s3:getaccelerateconfiguration + - s3:getbucketcors + - s3:getbucketnotification + - s3:getbucketwebsite + - s3:getreplicationconfiguration + - s3:getanalyticsconfiguration + - s3:getmetricsconfiguration + - s3:getinventoryconfiguration + - route53domains:getdomaindetail + - cloudtrail:gettrailstatus + +Contributors: + +- @zollman +- @robertoriv +- @hyperbolist +- @markofu +- @AngeloCiffa +- @jnbnyc +- @crruthe +- @nagwww +- @mikegrima +- @monkeysecurity + v0.7.0 (2016-09-21) =================== - PR #410/#405 - @zollman - Custom Watcher/Auditor Support. (Dynamic Loading) @@ -99,7 +177,7 @@ v0.6.0 (2016-08-29) - PR #386 - Shortening sessions from default value to 60 minutes. Setting Cookie HTTPONLY and SECURE flags. - PR #389 - Adding CloudTrail table, linked to itemrevision. (To be used by bananapeel rearchitecture.) - PR #390 - @ollytheninja - Adding export CSV button. -- PR #394 - @mikegrima - Saving exceptions to database table +- PR #394 - @mikegrima - Saving exceptions to database table - PR #402 - issue #401 - Adding new ELB Reference Policy ELBSecurityPolicy-2016-08 @@ -134,7 +212,7 @@ Important Notes: SESSION_COOKIE_SECURE=True SESSION_COOKIE_HTTPONLY=True PREFERRED_URL_SCHEME='https' - + REMEMBER_COOKIE_DURATION=timedelta(minutes=60) # Can make longer if you want remember_me to be useful REMEMBER_COOKIE_SECURE=True REMEMBER_COOKIE_HTTPONLY=True diff --git a/docs/configuration.rst b/docs/configuration.rst index ec6c4cb6a..e9845a41c 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -74,6 +74,7 @@ SM-ReadOnly "acm:describecertificate", "acm:listcertificates", "cloudtrail:describetrails", + "cloudtrail:gettrailstatus", "config:describeconfigrules", "config:describeconfigurationrecorders", "directconnect:describeconnections", @@ -145,14 +146,23 @@ SM-ReadOnly "route53:listhostedzones", "route53:listresourcerecordsets", "route53domains:listdomains", + "route53domains:getdomaindetail", + "s3:getaccelerateconfiguration", "s3:getbucketacl", + "s3:getbucketcors", "s3:getbucketlocation", "s3:getbucketlogging", + "s3:getbucketnotification", "s3:getbucketpolicy", "s3:getbuckettagging", "s3:getbucketversioning", + "s3:getbucketwebsite", "s3:getlifecycleconfiguration", "s3:listallmybuckets", + "s3:getreplicationconfiguration", + "s3:getanalyticsconfiguration", + "s3:getmetricsconfiguration", + "s3:getinventoryconfiguration", "ses:getidentityverificationattributes", "ses:listidentities", "ses:listverifiedemailaddresses", diff --git a/docs/contributing.rst b/docs/contributing.rst index ede8a6d32..367bbbe15 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -20,7 +20,7 @@ Please review the `Ubuntu Development Setup Instructions ` Development Setup Windows ======================== -Download VirtualBox, install Ubuntu, and then review the `Ubuntu Development Setup Instructions `_ to set up your Ubuntu VM for Security Monkey Development. +Please review the `Windows Development Setup Instructions `_ to set up Windows for Security Monkey development. Submitting changes ================== diff --git a/docs/dev_setup_osx.rst b/docs/dev_setup_osx.rst index 07bb7652d..a03ba8e8c 100644 --- a/docs/dev_setup_osx.rst +++ b/docs/dev_setup_osx.rst @@ -10,7 +10,16 @@ You will need to have the proper IAM Role configuration in place. See `Configur Additionally, see the boto documentation for more information: http://boto.readthedocs.org/en/latest/boto_config_tut.html -Install Brew (http://brew.sh) +Install XCode +========================== +XCode contains a number of tools that are required to install Security Monkey dependencies. This needs to be installed from the App Store (free download): +https://itunes.apple.com/us/app/xcode/id497799835?mt=12 + +After XCode is installed, you need to accept the XCode license agreement. To do that, run:: + + sudo xcodebuild -license # You will need to type in 'agree' + +Install Homebrew (http://brew.sh) ========================== Requirement - Xcode Command Line Tools (Popup - Just click Install):: @@ -95,6 +104,14 @@ Install Pip Requirements ========================== Pip will install all the dependencies into the current virtualenv. :: + # Note for El Capitan users and above: Apple has removed OpenSSL from OS X, which is a dependency + # of the cryptography library. OpenSSL gets installed with Postgres above. However, there are compiler + # path errors that result when trying to install the cryptography Python dependency. + # To resolve this, you need to run: + env LDFLAGS="-L$(brew --prefix openssl)/lib" CFLAGS="-I$(brew --prefix openssl)/include" python setup.py develop + # The above fully installs all the Python dependencies. + + # For OS X versions prior to El Capitan, run: python setup.py develop Init the Security Monkey DB @@ -171,11 +188,7 @@ Launch and Configure the WebStorm Editor ========================== We prefer the WebStorm IDE for developing with Dart: https://www.jetbrains.com/webstorm/. Webstorm requires the JDK to be installed. If you don't already have Java and the JDK installed, please download it here: http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html. -In addition to WebStorm, you will also need to have the Dart SDK installed. Please download and install the Dart suite (SDK and Dartium): - -**Note:** security_monkey is currently pinned to dart v1.12.1 and does not work with newer versions. Until we fix that, you'll need to download the dart sdk manually at https://www.dartlang.org/downloads/archive/ - -After we fix the issue, you will be able to use homebrew: +In addition to WebStorm, you will also need to have the Dart SDK installed. Please download and install the Dart suite (SDK and Dartium) via Homebrew:: $ brew tap dart-lang/dart $ brew install dart --with-content-shell --with-dartium @@ -209,7 +222,7 @@ This will add Amazon owned AWS accounts to security monkey. :: Add a user account ========================== -This will add a user account that can be used later to login to the web ui: +This will add a user account that can be used later to login to the web ui:: python manage.py create_user email@youremail.com Admin diff --git a/docs/dev_setup_windows.rst b/docs/dev_setup_windows.rst new file mode 100644 index 000000000..53ea1f9cc --- /dev/null +++ b/docs/dev_setup_windows.rst @@ -0,0 +1,268 @@ +************************************ +Development Setup on Windows +************************************ + +Please follow the instructions below for setting up the Security Monkey development environment on Windows 10. + +These instructions were created after consulting my install notes after recently getting a Windows 10 machine. If you're a Powershell guru, please feel free to send a PR to fix any errors. + +Windows Development +=================== +I'm pretty happy with development on Windows. Docker seems much easier to work with (No need for virtualbox). Gunicorn does not yet support Windows (Issue #524). Luckily, we don't need Gunicorn for local dev. Powershell is a worthy command line environment. If all else fails, use WSL (Windows Subsystem for Linux). + +AWS Credentials +========================== +You will need to have the proper IAM Role configuration in place. See `Configuration `_ for more details. Additionally, you will need to have IAM keys available within your environment variables. There are many ways to accomplish this. Please see Amazon's documentation for additional details: http://docs.aws.amazon.com/general/latest/gr/getting-aws-sec-creds.html. + +Additionally, see the boto documentation for more information: http://boto.readthedocs.org/en/latest/boto_config_tut.html + +Install Chocolatey +========================== + +Follow the instructions to install Chocolatey: + +https://chocolatey.org/install + + +Install Python +========================== + +Install python 2.7 with Chocolatey.:: + + choco install python2 + +Setup Powershell +========================== + +The following steps are a summary of the steps at http://www.tylerbutler.com/2012/05/how-to-install-python-pip-and-virtualenv-on-windows-with-powershell/ + +Execution Policy +----------------- + +You'll need to set the execution policy. There are a few options described at https://technet.microsoft.com/en-us/library/ee176961.aspx + +You'll need to run something like this:: + + Set-ExecutionPolicy RemoteSigned + +VirtualEnv +---------------- + +Install virtualenv and virtualenvwrapper from pypi:: + + pip install virtualenv + pip install virtualenvwrapper-powershell + +Try to import the powershell module:: + + Import-Module virtualenvwrapper + +At this point you may receive the following error:: + + Get-Content : Cannot find path 'Function:\TabExpansion' because it does not exist. + +You'll need to find and edit the file `virtualenvwrapperTabExpansion.psm1`. On line 12, replace `Get-Content Function:TabExpansion` with `Get-Content Function:TabExpansion2`. This should fix the import error. + +If the `~/.virtualenvs` folder wasn't created, do that now:: + + mkdir ~/.virtualenvs + +Automatically import the virtualenvwrapper module on powershell startup. +------------------------------------------------------------------------ + +In bash, you would typically edit your `~/.bashrc` to load modules and setup your environment. On Powershell, you'll use $profile. Powershell has a few different $profiles you can use. You can see them all with this command:: + + $profile | Format-List * -Force + AllUsersAllHosts : C:\Windows\System32\WindowsPowerShell\v1.0\profile.ps1 + AllUsersCurrentHost : C:\Windows\System32\WindowsPowerShell\v1.0\Microsoft.PowerShell_profile.ps1 + CurrentUserAllHosts : C:\Users\\Documents\WindowsPowerShell\profile.ps1 + CurrentUserCurrentHost : C:\Users\\Documents\WindowsPowerShell\Microsoft.PowerShell_profile.ps1 + Length : 77 + +If you're not sure, we're going to use `CurrentUserAllHosts`. If the file doesn't already exist, we can easily create it:: + + New-Item -Path $Profile.CurrentUserAllHosts -Type file -Force + +Now open it with a text editor and add this line:: + + Import-Module virtualenvwrapper + +All new powershell windows should have this module.:: + + Get-Command *virtualenv* + +Clone the Codebase +================== + +Navigate to wherever you like to mash on code and clone the repository. We'll use `~\Github` here.:: + + cd ~ + mkdir Github + cd Github + +If you don't already have git installed:: + + choco install git + +Clone security_monkey:: + + git clone git@github.com:Netflix/security_monkey.git + +Create a security_monkey virtualenv +=================================== + +You can use the powershell syntax:: + + New-VirtualEnvironment security_monkey + +Or use the aliased commands you're probably more familiar with:: + + mkvirtualenv security_monkey + +Before we attempt to install `setup.py`, let's grab a couple modules from pypi so we don't need to compile them.:: + + pip install cryptography + pip install bcrypt + +Install psycopg2 +---------------- + +This part seems a bit yucky. Let me know if you find a cleaner way. + +* Go to http://www.stickpeople.com/projects/python/win-psycopg/ +* Download the exe for your python version and processor architecture. I'll continue with `psycopg2-2.6.2.win-amd64-py2.7-pg9.5.3-release.exe` +* In powershell, ensure your virtualenv is activated and install the exe:: + + workon security_monkey + easy_install psycopg2-2.6.2.win-amd64-py2.7-pg9.5.3-release.exe + +Setting SECURITY_MONKEY_SETTINGS +-------------------------------- + +You set powershell environment variables with `$env:`:: + + $env:SECURITY_MONKEY_SETTINGS = "C:\Users\\...\GitHub\security_monkey\env-config\config-local.py" + +It might be a good idea to drop this into your $profile as well... + +Install Setup.py +---------------- + +With your virtualenv activated, this will install the security_monkey python module for dev:: + + cd ~/Github/security_monkey/ + workon security_monkey + python setup.py develop + +We should be able to run `manage.py` to see usage information: + + python manage.py + +Setup a development DB +---------------------- + +Instead of installing postgres, let's use docker for the DB. Windows has good docker support. You should be able to use Chocolatey, but I downloaded it directly from their website.:: + + choco install docker + +I actually downloaded the stable branch from here: https://docs.docker.com/docker-for-windows/ + +Once you have docker, pull a postgres container down. I'm using this one: https://hub.docker.com/r/library/postgres/ You should be able to start it with this command:: + + docker run --name some-postgres + +Kitematic is a nice UI tool for managing running containers. You can use it to set the postgres container to be reachable from localhost on 5432 and to set environment variables which the container uses to set the database name, username, password, etc. + +If you leave the DB paramaters at their default, you'll need to modify config-local.py:: + + SQLALCHEMY_DATABASE_URI = 'postgresql://postgres:mysecretpassword@localhost:5432/postgres' + +Install the security_monkey DB tables:: + + python manage.py db upgrade + +FYI - Navicat is a great tool for exploring the DB. + +Add Amazon Accounts +========================== +This will add Amazon owned AWS accounts to security monkey. :: + + python manage.py amazon_accounts + +Add a user account +========================== +This will add a user account that can be used later to login to the web ui: + + python manage.py create_user email@youremail.com Admin + +The first argument is the email address of the new user. The second parameter is the role and must be one of [anonymous, View, Comment, Justify, Admin]. + +Start the Security Monkey API +============================= +This starts the REST API that the Angular application will communicate with. :: + + python manage.py runserver + +Dart Development +---------------- + +Install the dart SDK:: + + choco install dart-sdk + +This will install a few tools in C:\tools. Let's install webstorm and configure it to use the dart-sdk:: + + choco install webstorm + +Open Webstorm and select the ~/Github/security_monkey/dart folder to open. We need webstorm to install the dart package. I believe it will popup and ask to install the dart package if you open the pubspec.yaml, or one of the dart files. Once the dart package is installed, go to File->Settings and select dart from the left column. + +* Check the box `Enable Dart Support ...` and provide the path `C:\tools\dart-sdk` +* Provide the path to dartium: `C:\tools\dartium\chrome.exe` + +Before we instruct webstorm to open ui.html with Dartium, we'll need to update `dart/lib/util/constants.dart`:: + + library security_monkey.constants; + ... + // LOCAL DEV + final String API_HOST = 'http://127.0.0.1:5000/api/1'; + //final bool REMOTE_AUTH = true; + + // Same Box + //final String API_HOST = '/api/1'; + final bool REMOTE_AUTH = false; + +You should now be able to use webstorm and dartium to work on the web ui. + +TODO: Determine if it makes sense to modify `security_monkey/__init__.py` to change the static_url path to the dart folder for webstorm development:: + + app = Flask(__name__, static_url_path='../dart/') + # does this work? + +Log into Security Monkey +========================== +Logging into Security Monkey is done by accessing the login page: ``http://127.0.0.1:8080/login``. Please note, that in the development environment, when you log in, you will be redirected to ``http://127.0.0.1/None``. This only occurs in the development environment. You will need to navigate to the WebStorm address and port (you can simply use WebStorm to re-open the page in Daritum). Once you are back in Dartium, you will be greeted with the main Security Monkey interface. + +Watch an AWS Account +========================== +After you have registered a user, logged in, and re-opened Dartium from WebStorm, you should be at the main Security Monkey interface. Once here, click on Settings and on the *+* to add a new AWS account to sync. + +Manually Run the Account Watchers +================================= +Run the watchers to put some data in the database. :: + + cd ~/Github/security_monkey/ + python manage.py run_change_reporter all + +You can also run an individual watcher:: + + python manage.py find_changes -a all -m all + python manage.py find_changes -a all -m iamrole + python manage.py find_changes -a "My Test Account" -m iamgroup + +You can run the auditors against the items currently in the database:: + + python manage.py audit_changes -a all -m redshift --send_report=False + +Next Steps +======================== +Continue reading the `Contributing `_ guide for additional instructions. diff --git a/docs/development.rst b/docs/development.rst index d55ad5728..1a15f6559 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -1,9 +1,9 @@ -********************** +====================== Development Guidelines -********************** +====================== Adding a Watcher ----------------- +================ Watchers are located in the `watchers <../security_monkey/watchers/>`_ directory. Some related watcher types are grouped together in common sub directories. An example would be IAM types. @@ -59,7 +59,7 @@ New Watchers may also require additional code: - If the api to access the system to be watched requires an explicit connection, connection functionality should be placed in the `sts_connect <../security_monkey/common/sts_connect.py>`_ module. Adding an Auditor ------------------- +----------------- A watcher may have one or more associated Auditors that will be run against all new or modified items to determine if there are any security issues. In order to be associated with a Watcher, the auditor class must override the index to match that of it's associated watcher. @@ -96,3 +96,121 @@ changed items returned by the watcher:: If an issue is found, the 'check_' method should call add_issue to save the issue to the database. + +Advanced Auditor Dependencies +----------------------------- +In some cases, an auditor needs information from technology types other than that of +the associated watcher to determine if there is a security risk. One example is the +determination of whether or not a route table is open to the internet. This requires +the ability to match a route gateway with the results of the VPC internet gateway returned +by the VPC watcher. The Auditor base class provides the method get_watcher_support_items() +to make the current results from one watcher available to another. In order to easily track +which watchers and auditors are dependent on each other, an additional configuration +is required in the in the Auditor class:: + + class SampleAuditor(Auditor): + index = Sample.index + i_am_singular = Sample.i_am_singular + i_am_plural = Sample.i_am_plural + support_watcher_indexes=[DependencyWatcher.index] + +Without this declaration the call to get_watcher_support_items() will fail. + +There are instances where auditor logic is dependent not just on the items from other watchers, +but also on the actual audit results. One example would be an IAM Group which was +configured to use an AWS managed policy. If the managed policy contained a security +risk, that risk would also be present in IAM Groups using this policy. The concept +of auditor hierarchies was introduces to manage this. + +The base Auditor object contains a method called get_auditor_support_items() that is similar +to get_watcher_support_items() except that in addition to the items returned by the watcher, +it also returns the latest audit results for each item. This introduces the risk of circular +dependencies because if AuditorA is dependent on AuditorB, in order to make AuditorB results +available when AuditorA is run: + +1. AuditorB must be run before AuditorA and +2. AuditorB cannot be dependent an AuditorA, nor may any dependencies of AuditorB be dependent on AuditorA + +In order to manage this, the the auditor class required a list of dependent auditors to be declared:: + + class SampleAuditor(Auditor): + index = Sample.index + i_am_singular = Sample.i_am_singular + i_am_plural = Sample.i_am_plural + support_auditor_indexes=[DependencyAuditor.index] + +Without this declaration the call to get_auditor_support_items() will fail. + +However, if any circular dependencies are detected the system will throw an exception with the the message at startup:: + + Detected circular dependency in support auditor {path of circular dependency} + +Linking to Auditor Dependencies +------------------------------- + +Typically, if an audit issue is dependent on another one, a the two should be linked: + +.. image:: images/linked_issue.png + +This can be achieved by the `Auditor <../../security_monkey/auditor.py>`_ link_to_support_item_issues() method. + +Custom Account Types +==================== +By default, Security Monkey runs against a basic AWS account but the custom account +framework allows the developer to either extend an AWS account with additional metadata +or to create a totally different account type to be monitored, such as an Active Directory +account. + +All account types extend the `AccountManager <../security_monkey/account_manager.py>`_ class and are located +in the `account_managers <../security_monkey/account_managers/>`_ directory. Account +types specific to an organization which are not intended to be contributed back to +the OSS community should be placed in the `account_managers/custom <../security_monkey/account_managers/custom>`_ directory. + +Data Structure +-------------- +The account contains five common fields: + +- name is the Security Monkey application defined name +- identifer is unique identifier of the account used to connect. For AWS accounts this would be the number +- active is a flag that determines whether to report on the account +- notes additional account information +- third_party AWS specific field that is used in Auditor._check_cross_account + +When creating a custom account type, additional fields may be added using the +account_manager.CustomFieldConfig objects which is used to display the fields on +the Account Settings page:: + + class CustomFieldConfig(object): + """ + Defines additional field types for custom account types + """ + def __init__(self, name, label, db_item, tool_tip, password=False): + super(CustomFieldConfig, self).__init__() + self.name = name + self.label = label + self.db_item = db_item + self.tool_tip = tool_tip + self.password = password + +Values created from this page are saved in the DB using the datastore.AccountTypeCustomValues +class is the db_item flag is True. + +Creating a Custom Account Type +------------------------------ +Custom account types must override three values: + +- account_type is a unique identifier for the type which is also used in the Watcher class to determine which watcher(s) to run against which account(s). +- identifier_label is used in the Account Settings page to display the label for the unique identifier for the account. +- identifier_tooltip is also used in the Account Settings page. + +The following overrides are optional: + +- compatable_account_types is a list that will cause watchers of these account types to also be run against the account. This is used when an account type overrides another account type to add additional data elements. +- custom_field_configs adds additional fields as described above +- def _load(self, account): this method is called to load custom fields from some third party datasource when the CustomFieldConfig.db_item field is defined as False + +Examples of these overrides are available at: + +- `Sample Active Directory Account Type <../security_monkey/account_managers/custom/sample_active_directory.py>`_ +- `Sample Active DB Extended AWS Account Type <../security_monkey/account_managers/custom/sample_db_extended_aws.py>`_ +- `Sample Active External Extended AWS Type <../security_monkey/account_managers/custom/sample_extended_aws.py>`_ diff --git a/docs/docker.rst b/docs/docker.rst new file mode 100644 index 000000000..6e0844fd2 --- /dev/null +++ b/docs/docker.rst @@ -0,0 +1,51 @@ +Docker Instructions +=================== + +The docker-compose.yml file describes the SecurityMonkey environment. This is intended for local development with the intention of deploying SecurityMonkey containers with a Docker Orchestration tool like Kubernetes. + +The Dockerfile builds SecurityMonkey into a container with several different entrypoints. These are for the different responsibilities SecurityMonkey has. +Also, the docker/nginx/Dockerfile file is used to build an NGINX container that will front the API, serve the static assets, and provide TLS. + +Quick Start: +------------ + Define your specific settings in **secmonkey.env** file. For example, this file will look like:: + + AWS_ACCESS_KEY_ID= + AWS_SECRET_ACCESS_KEY= + SECURITY_MONKEY_POSTGRES_HOST=postgres + SECURITY_MONKEY_FQDN=192.168.99.100 + + $ docker-compose build + ``this will locally build all the containers necessary`` + + $ docker-compose up -d postgres + ``this will start the database container`` + + $ docker-compose up -d init + ``this will start a container in which you canuse to setup the database, create users, and other manual configurations, see the below section for more info`` + + $ docker-compose up + ``this will bring up the remaining containers (scheduler and nginx)`` + +Commands: +--------- + + $ docker-compose build ``[api | scheduler | nginx | init]`` + + $ docker-compose up -d ``[postgres | api | scheduler | nginx | init]`` + +More Info: +---------- +:: + + $ docker-compose up -d init + +The init container is where the SecurityMonkey code is available for you to run manual configurations such as:: + + $ python manage.py create_user admin@example.com Admin + +and/or:: + + $ python manage.py add_account --number $account --name $name -r SecurityMonkey + +The init container provides a sandbox and is useful for local development. It is not required otherwise. diff --git a/docs/images/linked_issue.png b/docs/images/linked_issue.png new file mode 100644 index 000000000..bba45a1df Binary files /dev/null and b/docs/images/linked_issue.png differ diff --git a/docs/quickstart.rst b/docs/quickstart.rst index fc475353d..2f4ecc384 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -8,6 +8,8 @@ Docker Images Before we start, consider following the `docker instructions `_ . Docker helps simplify the process to get up and running. The docker images are not currently ready for production use, but are good enough to get up and running with an instance of security_monkey. +Local `docker instructions <./docker.html>`_ + Not into the docker thing? Keep reading. Setup IAM Roles @@ -86,6 +88,7 @@ Paste in this JSON with the name "SecurityMonkeyReadOnly": "acm:describecertificate", "acm:listcertificates", "cloudtrail:describetrails", + "cloudtrail:gettrailstatus", "config:describeconfigrules", "config:describeconfigurationrecorders", "directconnect:describeconnections", @@ -157,14 +160,23 @@ Paste in this JSON with the name "SecurityMonkeyReadOnly": "route53:listhostedzones", "route53:listresourcerecordsets", "route53domains:listdomains", + "route53domains:getdomaindetail", + "s3:getaccelerateconfiguration", "s3:getbucketacl", + "s3:getbucketcors", "s3:getbucketlocation", "s3:getbucketlogging", + "s3:getbucketnotification", "s3:getbucketpolicy", "s3:getbuckettagging", "s3:getbucketversioning", + "s3:getbucketwebsite", "s3:getlifecycleconfiguration", "s3:listallmybuckets", + "s3:getreplicationconfiguration", + "s3:getanalyticsconfiguration", + "s3:getmetricsconfiguration", + "s3:getinventoryconfiguration", "ses:getidentityverificationattributes", "ses:listidentities", "ses:listverifiedemailaddresses", @@ -453,7 +465,7 @@ Edit /usr/local/src/security_monkey/env-config/config-deploy.py: MAX_THREADS = 30 # SSO SETTINGS: - ACTIVE_PROVIDERS = [] # "ping" or "google" + ACTIVE_PROVIDERS = [] # "ping", "google" or "onelogin" PING_NAME = '' # Use to override the Ping name in the UI. PING_REDIRECT_URI = "{BASE}api/1/auth/ping".format(BASE=BASE_URL) @@ -468,6 +480,98 @@ Edit /usr/local/src/security_monkey/env-config/config-deploy.py: GOOGLE_AUTH_ENDPOINT = '' GOOGLE_SECRET = '' + ONELOGIN_APP_ID = '' # OneLogin App ID provider by your administrator + ONELOGIN_EMAIL_FIELD = 'User.email' # SAML attribute used to provide email address + ONELOGIN_DEFAULT_ROLE = 'View' # Default RBAC when user doesn't already exist + ONELOGIN_HTTPS = True # If using HTTPS strict mode will check the requests are HTTPS + ONELOGIN_SETTINGS = { + # If strict is True, then the Python Toolkit will reject unsigned + # or unencrypted messages if it expects them to be signed or encrypted. + # Also it will reject the messages if the SAML standard is not strictly + # followed. Destination, NameId, Conditions ... are validated too. + "strict": True, + + # Enable debug mode (outputs errors). + "debug": True, + + # Service Provider Data that we are deploying. + "sp": { + # Identifier of the SP entity (must be a URI) + "entityId": "{BASE}metadata/".format(BASE=BASE_URL), + # Specifies info about where and how the message MUST be + # returned to the requester, in this case our SP. + "assertionConsumerService": { + # URL Location where the from the IdP will be returned + "url": "{BASE}api/1/auth/onelogin?acs".format(BASE=BASE_URL), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports this endpoint for the + # HTTP-POST binding only. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST" + }, + # If you need to specify requested attributes, set a + # attributeConsumingService. nameFormat, attributeValue and + # friendlyName can be omitted + #"attributeConsumingService": { + # "ServiceName": "SP test", + # "serviceDescription": "Test Service", + # "requestedAttributes": [ + # { + # "name": "", + # "isRequired": False, + # "nameFormat": "", + # "friendlyName": "", + # "attributeValue": "" + # } + # ] + #}, + # Specifies info about where and how the message MUST be + # returned to the requester, in this case our SP. + "singleLogoutService": { + # URL Location where the from the IdP will be returned + "url": "{BASE}api/1/auth/onelogin?sls".format(BASE=BASE_URL), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports the HTTP-Redirect binding + # only for this endpoint. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" + }, + # Specifies the constraints on the name identifier to be used to + # represent the requested subject. + # Take a look on src/onelogin/saml2/constants.py to see the NameIdFormat that are supported. + "NameIDFormat": "urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified", + # Usually x509cert and privateKey of the SP are provided by files placed at + # the certs folder. But we can also provide them with the following parameters + "x509cert": "", + "privateKey": "" + }, + + # Identity Provider Data that we want connected with our SP. + "idp": { + # Identifier of the IdP entity (must be a URI) + "entityId": "https://app.onelogin.com/saml/metadata/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), + # SSO endpoint info of the IdP. (Authentication Request protocol) + "singleSignOnService": { + # URL Target of the IdP where the Authentication Request Message + # will be sent. + "url": "https://app.onelogin.com/trust/saml2/http-post/sso/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports the HTTP-Redirect binding + # only for this endpoint. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" + }, + # SLO endpoint info of the IdP. + "singleLogoutService": { + # URL Location of the IdP where SLO Request will be sent. + "url": "https://app.onelogin.com/trust/saml2/http-redirect/slo/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports the HTTP-Redirect binding + # only for this endpoint. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" + }, + # Public x509 certificate of the IdP + "x509cert": "" + } + } + from datetime import timedelta PERMANENT_SESSION_LIFETIME=timedelta(minutes=60) # Will logout users after period of inactivity. SESSION_REFRESH_EACH_REQUEST=True diff --git a/env-config/config-deploy.py b/env-config/config-deploy.py index f5c5fb0ba..bb4f7b522 100644 --- a/env-config/config-deploy.py +++ b/env-config/config-deploy.py @@ -107,7 +107,7 @@ MAX_THREADS = 30 # SSO SETTINGS: -ACTIVE_PROVIDERS = [] # "ping" or "google" +ACTIVE_PROVIDERS = [] # "ping", "google" or "onelogin" PING_NAME = '' # Use to override the Ping name in the UI. PING_REDIRECT_URI = "{BASE}api/1/auth/ping".format(BASE=BASE_URL) @@ -122,6 +122,98 @@ GOOGLE_AUTH_ENDPOINT = '' GOOGLE_SECRET = '' +ONELOGIN_APP_ID = '' # OneLogin App ID provider by your administrator +ONELOGIN_EMAIL_FIELD = 'User.email' # SAML attribute used to provide email address +ONELOGIN_DEFAULT_ROLE = 'View' # Default RBAC when user doesn't already exist +ONELOGIN_HTTPS = True # If using HTTPS strict mode will check the requests are HTTPS +ONELOGIN_SETTINGS = { + # If strict is True, then the Python Toolkit will reject unsigned + # or unencrypted messages if it expects them to be signed or encrypted. + # Also it will reject the messages if the SAML standard is not strictly + # followed. Destination, NameId, Conditions ... are validated too. + "strict": True, + + # Enable debug mode (outputs errors). + "debug": True, + + # Service Provider Data that we are deploying. + "sp": { + # Identifier of the SP entity (must be a URI) + "entityId": "{BASE}metadata/".format(BASE=BASE_URL), + # Specifies info about where and how the message MUST be + # returned to the requester, in this case our SP. + "assertionConsumerService": { + # URL Location where the from the IdP will be returned + "url": "{BASE}api/1/auth/onelogin?acs".format(BASE=BASE_URL), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports this endpoint for the + # HTTP-POST binding only. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST" + }, + # If you need to specify requested attributes, set a + # attributeConsumingService. nameFormat, attributeValue and + # friendlyName can be omitted + #"attributeConsumingService": { + # "ServiceName": "SP test", + # "serviceDescription": "Test Service", + # "requestedAttributes": [ + # { + # "name": "", + # "isRequired": False, + # "nameFormat": "", + # "friendlyName": "", + # "attributeValue": "" + # } + # ] + #}, + # Specifies info about where and how the message MUST be + # returned to the requester, in this case our SP. + "singleLogoutService": { + # URL Location where the from the IdP will be returned + "url": "{BASE}api/1/auth/onelogin?sls".format(BASE=BASE_URL), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports the HTTP-Redirect binding + # only for this endpoint. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" + }, + # Specifies the constraints on the name identifier to be used to + # represent the requested subject. + # Take a look on src/onelogin/saml2/constants.py to see the NameIdFormat that are supported. + "NameIDFormat": "urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified", + # Usually x509cert and privateKey of the SP are provided by files placed at + # the certs folder. But we can also provide them with the following parameters + "x509cert": "", + "privateKey": "" + }, + + # Identity Provider Data that we want connected with our SP. + "idp": { + # Identifier of the IdP entity (must be a URI) + "entityId": "https://app.onelogin.com/saml/metadata/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), + # SSO endpoint info of the IdP. (Authentication Request protocol) + "singleSignOnService": { + # URL Target of the IdP where the Authentication Request Message + # will be sent. + "url": "https://app.onelogin.com/trust/saml2/http-post/sso/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports the HTTP-Redirect binding + # only for this endpoint. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" + }, + # SLO endpoint info of the IdP. + "singleLogoutService": { + # URL Location of the IdP where SLO Request will be sent. + "url": "https://app.onelogin.com/trust/saml2/http-redirect/slo/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports the HTTP-Redirect binding + # only for this endpoint. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" + }, + # Public x509 certificate of the IdP + "x509cert": "" + } +} + from datetime import timedelta PERMANENT_SESSION_LIFETIME=timedelta(minutes=60) SESSION_REFRESH_EACH_REQUEST=True @@ -132,3 +224,9 @@ REMEMBER_COOKIE_DURATION=timedelta(minutes=60) # Can make longer if you want remember_me to be useful. REMEMBER_COOKIE_SECURE=True REMEMBER_COOKIE_HTTPONLY=True + +# Apscheduler Configurations +# Length of time, in seconds, before a scheduled job is cancelled due to thread contention or other issues +MISFIRE_GRACE_TIME=30 +# Delay, in seconds, until reporter starts +REPORTER_START_DELAY=10 \ No newline at end of file diff --git a/env-config/config-docker.py b/env-config/config-docker.py new file mode 100644 index 000000000..20127e133 --- /dev/null +++ b/env-config/config-docker.py @@ -0,0 +1,253 @@ +# Copyright 2014 Netflix, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Insert any config items here. +# This will be fed into Flask/SQLAlchemy inside security_monkey/__init__.py + +import os + + +def env_to_bool(input): + """ + Must change String from environment variable into Boolean + defaults to True + """ + if isinstance(input, str): + if input == 'False': + return False + else: + return True + else: + return input + + +LOG_CFG = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'standard': { + 'format': '%(asctime)s %(levelname)s: %(message)s ' + '[in %(pathname)s:%(lineno)d]' + } + }, + 'handlers': { + 'file': { + 'class': 'logging.handlers.RotatingFileHandler', + 'level': 'DEBUG', + 'formatter': 'standard', + 'filename': '/var/log/security_monkey/securitymonkey.log', + 'maxBytes': 10485760, + 'backupCount': 100, + 'encoding': 'utf8' + }, + 'console': { + 'class': 'logging.StreamHandler', + 'level': 'DEBUG', + 'formatter': 'standard', + 'stream': 'ext://sys.stdout' + } + }, + 'loggers': { + 'security_monkey': { + 'handlers': ['console'], + 'level': os.getenv('SM_CONSOLE_LOG_LEVEL', 'DEBUG') + }, + 'apscheduler': { + 'handlers': ['console'], + 'level': os.getenv('SM_APPSCHEDULER_LOG_LEVEL', 'INFO') + } + } +} + +SQLALCHEMY_DATABASE_URI = 'postgresql://%s:%s@%s:%d/%s' % ( + os.getenv('SECURITY_MONKEY_POSTGRES_USER', 'postgres'), + os.getenv('SECURITY_MONKEY_POSTGRES_PASSWORD', 'securitymonkeypassword'), + os.getenv('SECURITY_MONKEY_POSTGRES_HOST', 'localhost'), + os.getenv('SECURITY_MONKEY_POSTGRES_PORT', 5432), + os.getenv('SECURITY_MONKEY_POSTGRES_DATABASE', 'secmonkey') +) + +# print postgres +# print SQLALCHEMY_DATABASE_URI + +SQLALCHEMY_POOL_SIZE = 50 +SQLALCHEMY_MAX_OVERFLOW = 15 +ENVIRONMENT = 'ec2' +USE_ROUTE53 = False +FQDN = os.getenv('SECURITY_MONKEY_FQDN', 'ec2-XX-XXX-XXX-XXX.compute-1.amazonaws.com') +API_PORT = '5000' +WEB_PORT = '443' +WEB_PATH = '/static/ui.html' +FRONTED_BY_NGINX = True +NGINX_PORT = '443' +BASE_URL = 'https://{}/'.format(FQDN) + +SECRET_KEY = '' + +MAIL_DEFAULT_SENDER = os.getenv('SECURITY_MONKEY_EMAIL_DEFAULT_SENDER', 'securitymonkey@example.com') +SECURITY_REGISTERABLE = True +SECURITY_CONFIRMABLE = False +SECURITY_RECOVERABLE = False +SECURITY_PASSWORD_HASH = 'bcrypt' +SECURITY_PASSWORD_SALT = '' +SECURITY_TRACKABLE = True + +SECURITY_POST_LOGIN_VIEW = BASE_URL +SECURITY_POST_REGISTER_VIEW = BASE_URL +SECURITY_POST_CONFIRM_VIEW = BASE_URL +SECURITY_POST_RESET_VIEW = BASE_URL +SECURITY_POST_CHANGE_VIEW = BASE_URL + +# This address gets all change notifications (i.e. 'securityteam@example.com') +SECURITY_TEAM_EMAIL = os.getenv('SECURITY_MONKEY_SECURITY_TEAM_EMAIL', []) + +# These are only required if using SMTP instead of SES +EMAILS_USE_SMTP = env_to_bool(os.getenv('SECURITY_MONKEY_SMTP', True)) # Otherwise, Use SES +SES_REGION = os.getenv('SECURITY_MONKEY_SES_REGION', 'us-east-1') +MAIL_SERVER = os.getenv('SECURITY_MONKEY_EMAIL_SERVER', 'smtp.example.com') +MAIL_PORT = 465 +MAIL_USE_SSL = True +MAIL_USERNAME = os.getenv('SECURITY_MONKEY_EMAIL_USERNAME', 'username') +MAIL_PASSWORD = os.getenv('SECURITY_MONKEY_EMAIL_PASSWORD', 'password') + +WTF_CSRF_ENABLED = env_to_bool(os.getenv('SM_WTF_CSRF_ENABLED', True)) +# Checks Referer Header. Set to False for API access. +WTF_CSRF_SSL_STRICT = env_to_bool(os.getenv('SM_WTF_CSRF_SSL_STRICT', True)) +WTF_CSRF_METHODS = ['DELETE', 'POST', 'PUT', 'PATCH'] + +# "NONE", "SUMMARY", or "FULL" +SECURITYGROUP_INSTANCE_DETAIL = 'FULL' + +# Threads used by the scheduler. +# You will likely need at least one core thread for every account being monitored. +CORE_THREADS = 25 +MAX_THREADS = 30 + +# SSO SETTINGS: +ACTIVE_PROVIDERS = [] # "ping", "google" or "onelogin" + +PING_NAME = '' # Use to override the Ping name in the UI. +PING_REDIRECT_URI = "{BASE}api/1/auth/ping".format(BASE=BASE_URL) +PING_CLIENT_ID = '' # Provided by your administrator +PING_AUTH_ENDPOINT = '' # Often something ending in authorization.oauth2 +PING_ACCESS_TOKEN_URL = '' # Often something ending in token.oauth2 +PING_USER_API_URL = '' # Often something ending in idp/userinfo.openid +PING_JWKS_URL = '' # Often something ending in JWKS +PING_SECRET = '' # Provided by your administrator + +GOOGLE_CLIENT_ID = '' +GOOGLE_AUTH_ENDPOINT = '' +GOOGLE_SECRET = '' + +ONELOGIN_APP_ID = '' # OneLogin App ID provider by your administrator +ONELOGIN_EMAIL_FIELD = 'User.email' # SAML attribute used to provide email address +ONELOGIN_DEFAULT_ROLE = 'View' # Default RBAC when user doesn't already exist +ONELOGIN_HTTPS = True # If using HTTPS strict mode will check the requests are HTTPS +ONELOGIN_SETTINGS = { + # If strict is True, then the Python Toolkit will reject unsigned + # or unencrypted messages if it expects them to be signed or encrypted. + # Also it will reject the messages if the SAML standard is not strictly + # followed. Destination, NameId, Conditions ... are validated too. + "strict": True, + + # Enable debug mode (outputs errors). + "debug": True, + + # Service Provider Data that we are deploying. + "sp": { + # Identifier of the SP entity (must be a URI) + "entityId": "{BASE}metadata/".format(BASE=BASE_URL), + # Specifies info about where and how the message MUST be + # returned to the requester, in this case our SP. + "assertionConsumerService": { + # URL Location where the from the IdP will be returned + "url": "{BASE}api/1/auth/onelogin?acs".format(BASE=BASE_URL), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports this endpoint for the + # HTTP-POST binding only. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST" + }, + # If you need to specify requested attributes, set a + # attributeConsumingService. nameFormat, attributeValue and + # friendlyName can be omitted + #"attributeConsumingService": { + # "ServiceName": "SP test", + # "serviceDescription": "Test Service", + # "requestedAttributes": [ + # { + # "name": "", + # "isRequired": False, + # "nameFormat": "", + # "friendlyName": "", + # "attributeValue": "" + # } + # ] + #}, + # Specifies info about where and how the message MUST be + # returned to the requester, in this case our SP. + "singleLogoutService": { + # URL Location where the from the IdP will be returned + "url": "{BASE}api/1/auth/onelogin?sls".format(BASE=BASE_URL), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports the HTTP-Redirect binding + # only for this endpoint. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" + }, + # Specifies the constraints on the name identifier to be used to + # represent the requested subject. + # Take a look on src/onelogin/saml2/constants.py to see the NameIdFormat that are supported. + "NameIDFormat": "urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified", + # Usually x509cert and privateKey of the SP are provided by files placed at + # the certs folder. But we can also provide them with the following parameters + "x509cert": "", + "privateKey": "" + }, + + # Identity Provider Data that we want connected with our SP. + "idp": { + # Identifier of the IdP entity (must be a URI) + "entityId": "https://app.onelogin.com/saml/metadata/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), + # SSO endpoint info of the IdP. (Authentication Request protocol) + "singleSignOnService": { + # URL Target of the IdP where the Authentication Request Message + # will be sent. + "url": "https://app.onelogin.com/trust/saml2/http-post/sso/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports the HTTP-Redirect binding + # only for this endpoint. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" + }, + # SLO endpoint info of the IdP. + "singleLogoutService": { + # URL Location of the IdP where SLO Request will be sent. + "url": "https://app.onelogin.com/trust/saml2/http-redirect/slo/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports the HTTP-Redirect binding + # only for this endpoint. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" + }, + # Public x509 certificate of the IdP + "x509cert": "" + } +} + +from datetime import timedelta +PERMANENT_SESSION_LIFETIME=timedelta(minutes=60) +SESSION_REFRESH_EACH_REQUEST=True +SESSION_COOKIE_SECURE=True +SESSION_COOKIE_HTTPONLY=True +PREFERRED_URL_SCHEME='https' + +REMEMBER_COOKIE_DURATION=timedelta(minutes=60) # Can make longer if you want remember_me to be useful. +REMEMBER_COOKIE_SECURE=True +REMEMBER_COOKIE_HTTPONLY=True diff --git a/env-config/config-local.py b/env-config/config-local.py index 9680f7d13..2001ca273 100644 --- a/env-config/config-local.py +++ b/env-config/config-local.py @@ -102,7 +102,7 @@ SECURITYGROUP_INSTANCE_DETAIL = 'FULL' # SSO SETTINGS: -ACTIVE_PROVIDERS = [] # "ping" or "google" +ACTIVE_PROVIDERS = [] # "ping", "google" or "onelogin" PING_NAME = '' # Use to override the Ping name in the UI. PING_REDIRECT_URI = "http://{FQDN}:{PORT}/api/1/auth/ping".format(FQDN=FQDN, PORT=WEB_PORT) @@ -116,3 +116,95 @@ GOOGLE_CLIENT_ID = '' GOOGLE_AUTH_ENDPOINT = '' GOOGLE_SECRET = '' + +ONELOGIN_APP_ID = '' # OneLogin App ID provider by your administrator +ONELOGIN_EMAIL_FIELD = 'User.email' # SAML attribute used to provide email address +ONELOGIN_DEFAULT_ROLE = 'View' # Default RBAC when user doesn't already exist +ONELOGIN_HTTPS = True # If using HTTPS strict mode will check the requests are HTTPS +ONELOGIN_SETTINGS = { + # If strict is True, then the Python Toolkit will reject unsigned + # or unencrypted messages if it expects them to be signed or encrypted. + # Also it will reject the messages if the SAML standard is not strictly + # followed. Destination, NameId, Conditions ... are validated too. + "strict": True, + + # Enable debug mode (outputs errors). + "debug": True, + + # Service Provider Data that we are deploying. + "sp": { + # Identifier of the SP entity (must be a URI) + "entityId": "http://{FQDN}:{PORT}/metadata/".format(FQDN=FQDN, PORT=WEB_PORT), + # Specifies info about where and how the message MUST be + # returned to the requester, in this case our SP. + "assertionConsumerService": { + # URL Location where the from the IdP will be returned + "url": "http://{FQDN}:{PORT}/api/1/auth/onelogin?acs".format(FQDN=FQDN, PORT=WEB_PORT), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports this endpoint for the + # HTTP-POST binding only. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST" + }, + # If you need to specify requested attributes, set a + # attributeConsumingService. nameFormat, attributeValue and + # friendlyName can be omitted + #"attributeConsumingService": { + # "ServiceName": "SP test", + # "serviceDescription": "Test Service", + # "requestedAttributes": [ + # { + # "name": "", + # "isRequired": False, + # "nameFormat": "", + # "friendlyName": "", + # "attributeValue": "" + # } + # ] + #}, + # Specifies info about where and how the message MUST be + # returned to the requester, in this case our SP. + "singleLogoutService": { + # URL Location where the from the IdP will be returned + "url": "http://{FQDN}:{PORT}/api/1/auth/onelogin?sls".format(FQDN=FQDN, PORT=WEB_PORT), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports the HTTP-Redirect binding + # only for this endpoint. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" + }, + # Specifies the constraints on the name identifier to be used to + # represent the requested subject. + # Take a look on src/onelogin/saml2/constants.py to see the NameIdFormat that are supported. + "NameIDFormat": "urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified", + # Usually x509cert and privateKey of the SP are provided by files placed at + # the certs folder. But we can also provide them with the following parameters + "x509cert": "", + "privateKey": "" + }, + + # Identity Provider Data that we want connected with our SP. + "idp": { + # Identifier of the IdP entity (must be a URI) + "entityId": "https://app.onelogin.com/saml/metadata/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), + # SSO endpoint info of the IdP. (Authentication Request protocol) + "singleSignOnService": { + # URL Target of the IdP where the Authentication Request Message + # will be sent. + "url": "https://app.onelogin.com/trust/saml2/http-post/sso/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports the HTTP-Redirect binding + # only for this endpoint. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" + }, + # SLO endpoint info of the IdP. + "singleLogoutService": { + # URL Location of the IdP where SLO Request will be sent. + "url": "https://app.onelogin.com/trust/saml2/http-redirect/slo/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports the HTTP-Redirect binding + # only for this endpoint. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" + }, + # Public x509 certificate of the IdP + "x509cert": "" + } +} diff --git a/manage.py b/manage.py index 002ea14fe..e1582604c 100644 --- a/manage.py +++ b/manage.py @@ -19,7 +19,6 @@ from security_monkey import app, db from security_monkey.common.route53 import Route53Service -from gunicorn.app.base import Application from flask.ext.migrate import Migrate, MigrateCommand @@ -31,6 +30,16 @@ from security_monkey.datastore import Account from security_monkey.watcher import watcher_registry +try: + from gunicorn.app.base import Application + GUNICORN = True +except ImportError: + # Gunicorn does not yet support Windows. + # See issue #524. https://github.com/benoitc/gunicorn/issues/524 + # For dev on Windows, make this an optional import. + print('Could not import gunicorn, skipping.') + GUNICORN = False + manager = Manager(app) migrate = Migrate(app, db) @@ -71,6 +80,21 @@ def audit_changes(accounts, monitors, send_report): sm_audit_changes(account_names, monitor_names, send_report) +@manager.option('-a', '--accounts', dest='accounts', type=unicode, default=u'all') +@manager.option('-m', '--monitors', dest='monitors', type=unicode, default=u'all') +def delete_unjustified_issues(accounts, monitors): + """ Allows us to delete unjustified issues. """ + monitor_names = _parse_tech_names(monitors) + account_names = _parse_accounts(accounts) + from security_monkey.datastore import ItemAudit + # ItemAudit.query.filter_by(justified=False).delete() + issues = ItemAudit.query.filter_by(justified=False).all() + for issue in issues: + del issue.sub_items[:] + db.session.delete(issue) + db.session.commit() + + @manager.option('-a', '--accounts', dest='accounts', type=unicode, default=u'all') @manager.option('-m', '--monitors', dest='monitors', type=unicode, default=u'all') @manager.option('-o', '--outputfolder', dest='outputfolder', type=unicode, default=u'backups') @@ -116,13 +140,20 @@ def amazon_accounts(): """ Pre-populates standard AWS owned accounts """ import os import json - from security_monkey.datastore import Account + from security_monkey.datastore import Account, AccountType data_file = os.path.join(os.path.dirname(__file__), "data", "aws_accounts.json") data = json.load(open(data_file, 'r')) app.logger.info('Adding / updating Amazon owned accounts') try: + account_type_result = AccountType.query.filter(AccountType.name == 'AWS').first() + if not account_type_result: + account_type_result = AccountType(name='AWS') + db.session.add(account_type_result) + db.session.commit() + db.session.refresh(account_type_result) + for group, info in data.items(): for aws_account in info['accounts']: acct_name = "{group} ({region})".format(group=group, region=aws_account['region']) @@ -134,6 +165,8 @@ def amazon_accounts(): app.logger.debug(' Updating account {0}'.format(acct_name)) account.number = aws_account['account_id'] + account.identifier = aws_account['account_id'] + account.account_type_id = account_type_result.id account.active = False account.third_party = True account.name = acct_name @@ -154,15 +187,25 @@ def amazon_accounts(): @manager.option('-n', '--name', dest='name', type=unicode, required=True) @manager.option('-s', '--s3name', dest='s3_name', type=unicode, default=u'') @manager.option('-o', '--notes', dest='notes', type=unicode, default=u'') +@manager.option('-y', '--type', dest='account_type', type=unicode, default=u'AWS') @manager.option('-r', '--rolename', dest='role_name', type=unicode, default=u'SecurityMonkey') @manager.option('-f', '--force', dest='force', help='Override existing accounts', action='store_true') -def add_account(number, third_party, name, s3_name, active, notes, role_name, force): - from security_monkey.common.utils import add_account - res = add_account(number, third_party, name, s3_name, active, notes, role_name, force) - if res: - app.logger.info('Successfully added account {}'.format(name)) +def add_account(number, third_party, name, s3_name, active, notes, account_type, role_name, force): + from security_monkey.account_manager import account_registry + account_manager = account_registry.get(account_type)() + account = account_manager.lookup_account_by_identifier(number) + if account: + if force: + account_manager.update(account.id, account_type, name, active, + third_party, notes, number, + custom_fields={ 's3_name': s3_name, 'role_name': role_name }) + else: + app.logger.info('Account with id {} already exists'.format(number)) else: - app.logger.info('Account with id {} already exists'.format(number)) + account_manager.create(account_type, name, active, third_party, notes, number, + custom_fields={ 's3_name': s3_name, 'role_name': role_name }) + + db.session.close() @manager.command @@ -242,17 +285,20 @@ def handle(self, app, *args, **kwargs): workers = kwargs['workers'] address = kwargs['address'] - class FlaskApplication(Application): - def init(self, parser, opts, args): - return { - 'bind': address, - 'workers': workers - } + if not GUNICORN: + print('GUNICORN not installed. Try `runserver` to use the Flask debug server instead.') + else: + class FlaskApplication(Application): + def init(self, parser, opts, args): + return { + 'bind': address, + 'workers': workers + } - def load(self): - return app + def load(self): + return app - FlaskApplication().run() + FlaskApplication().run() if __name__ == "__main__": diff --git a/migrations/versions/1c847ae1209a_.py b/migrations/versions/1c847ae1209a_.py new file mode 100644 index 000000000..b55cf64b6 --- /dev/null +++ b/migrations/versions/1c847ae1209a_.py @@ -0,0 +1,26 @@ +"""Increase note max size + +Revision ID: 1c847ae1209a +Revises: bbd695323107 +Create Date: 2016-07-29 11:18:00.550197 + +""" + +# revision identifiers, used by Alembic. +revision = '1c847ae1209a' +down_revision = '2ce75615b24d' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.alter_column('itemaudit', 'notes', type_=sa.VARCHAR(1024), existing_type=sa.VARCHAR(length=512), nullable=True) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.alter_column('itemaudit', 'notes', type_=sa.VARCHAR(512), existing_type=sa.VARCHAR(length=1024), nullable=True) + ### end Alembic commands ### diff --git a/migrations/versions/2ce75615b24d_.py b/migrations/versions/2ce75615b24d_.py new file mode 100644 index 000000000..a4c549e7e --- /dev/null +++ b/migrations/versions/2ce75615b24d_.py @@ -0,0 +1,40 @@ +"""Adding index to region. Dropping item.cloud. + +Revision ID: 2ce75615b24d +Revises: d08d0b37788a +Create Date: 2016-11-15 20:31:27.393066 + +""" + +# revision identifiers, used by Alembic. +revision = '2ce75615b24d' +down_revision = 'd08d0b37788a' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(u'exceptions_tech_id_fkey', 'exceptions', type_='foreignkey') + op.drop_constraint(u'exceptions_item_id_fkey', 'exceptions', type_='foreignkey') + op.drop_constraint(u'exceptions_account_id_fkey', 'exceptions', type_='foreignkey') + op.create_foreign_key(None, 'exceptions', 'account', ['account_id'], ['id'], ondelete='CASCADE') + op.create_foreign_key(None, 'exceptions', 'item', ['item_id'], ['id'], ondelete='CASCADE') + op.create_foreign_key(None, 'exceptions', 'technology', ['tech_id'], ['id'], ondelete='CASCADE') + op.create_index('ix_item_region', 'item', ['region'], unique=False) + op.drop_column('item', 'cloud') + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('item', sa.Column('cloud', sa.VARCHAR(length=32), autoincrement=False, nullable=True)) + op.drop_index('ix_item_region', table_name='item') + op.drop_constraint(u'exceptions_tech_id_fkey', 'exceptions', type_='foreignkey') + op.drop_constraint(u'exceptions_item_id_fkey', 'exceptions', type_='foreignkey') + op.drop_constraint(u'exceptions_account_id_fkey', 'exceptions', type_='foreignkey') + op.create_foreign_key(u'exceptions_account_id_fkey', 'exceptions', 'account', ['account_id'], ['id']) + op.create_foreign_key(u'exceptions_item_id_fkey', 'exceptions', 'item', ['item_id'], ['id']) + op.create_foreign_key(u'exceptions_tech_id_fkey', 'exceptions', 'technology', ['tech_id'], ['id']) + ### end Alembic commands ### \ No newline at end of file diff --git a/migrations/versions/ad23a56abf25_.py b/migrations/versions/ad23a56abf25_.py new file mode 100644 index 000000000..61047f006 --- /dev/null +++ b/migrations/versions/ad23a56abf25_.py @@ -0,0 +1,31 @@ +"""Ability to link issues to other tech type items + +Revision ID: ad23a56abf25 +Revises: 67ea2aac5ea0 +Create Date: 2016-02-23 18:52:45.024716 + +""" + +# revision identifiers, used by Alembic. +revision = 'ad23a56abf25' +down_revision = '1a863bd1acb1' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('issue_item_association', + sa.Column('super_issue_id', sa.Integer(), nullable=True), + sa.Column('sub_item_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['sub_item_id'], ['item.id'], ), + sa.ForeignKeyConstraint(['super_issue_id'], ['itemaudit.id'], ) + ) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_table('issue_item_association') + ### end Alembic commands ### diff --git a/migrations/versions/d08d0b37788a_.py b/migrations/versions/d08d0b37788a_.py new file mode 100644 index 000000000..33916c482 --- /dev/null +++ b/migrations/versions/d08d0b37788a_.py @@ -0,0 +1,84 @@ +"""Enable custom account types + +Revision ID: d08d0b37788a +Revises: 0ae4ef82b244 +Create Date: 2016-05-19 18:36:18.693108 + +""" + +# revision identifiers, used by Alembic. +revision = 'd08d0b37788a' +down_revision = 'ad23a56abf25' + +from alembic import op +import sqlalchemy as sa + +from sqlalchemy.orm import sessionmaker +from sqlalchemy.ext.declarative import declarative_base + + +Session = sessionmaker() +Base = declarative_base() + + +class Account(Base): + __tablename__ = 'account' + id = sa.Column(sa.Integer, primary_key=True) + account_type_id = sa.Column(sa.Integer()) + number = sa.Column(sa.String()) + identifier = sa.Column(sa.String()) + + +class AccountType(Base): + __tablename__ = 'account_type' + id = sa.Column(sa.Integer, primary_key=True) + name = sa.Column('name', sa.String(length=80)) + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('account_type', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=80), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + op.create_table('account_type_values', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=64), nullable=True), + sa.Column('value', sa.String(length=256), nullable=True), + sa.Column('account_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['account_id'], ['account.id'], ), + sa.PrimaryKeyConstraint('id') + ) + + bind = op.get_bind() + session = Session(bind=bind) + + account_type = AccountType(name='AWS') + session.add(account_type) + session.commit() + + op.add_column(u'account', sa.Column('account_type_id', sa.Integer(), nullable=False, server_default=str(account_type.id))) + op.add_column(u'account', sa.Column('identifier', sa.String(length=256), nullable=True)) + op.create_foreign_key('account_account_type_id_fkey', 'account', 'account_type', ['account_type_id'], ['id']) + op.create_unique_constraint('account_name_uc', 'account', ['name']) + ### end Alembic commands ### + + accounts = session.query(Account).all() + for account in accounts: + account.identifier = account.number + session.add(account) + + session.commit() + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint('account_account_type_id_fkey', 'account', type_='foreignkey') + op.drop_column(u'account', 'identifier') + op.drop_column(u'account', 'account_type_id') + op.drop_table('account_type_values') + op.drop_table('account_type') + op.drop_constraint('account_name_uc', 'account', type_='unique') + ### end Alembic commands ### diff --git a/scripts/secmonkey_auto_install.sh b/scripts/secmonkey_auto_install.sh index 679b807db..bc63c9f72 100755 --- a/scripts/secmonkey_auto_install.sh +++ b/scripts/secmonkey_auto_install.sh @@ -281,7 +281,7 @@ install_post () install_pre () { - sudo apt-get update && sudo apt-get install -y python-pip python-dev python-psycopg2 libpq-dev nginx supervisor git postgresql-client libyaml-dev libffi-dev + sudo apt-get update && sudo apt-get install -y python-pip python-dev python-psycopg2 libpq-dev nginx supervisor git postgresql-client libyaml-dev libffi-dev libxml2-dev libxmlsec1-dev if (($db)) # Checking if the $db variable is an arithmetic operator then [[ $db =~ 127.0.0.1 ]] && install_post @@ -447,7 +447,7 @@ CORE_THREADS = 25 MAX_THREADS = 30 # SSO SETTINGS: -ACTIVE_PROVIDERS = [] # "ping" or "google" +ACTIVE_PROVIDERS = [] # "ping", "google" or "onelogin" PING_NAME = '' # Use to override the Ping name in the UI. PING_REDIRECT_URI = "{BASE}api/1/auth/ping".format(BASE=BASE_URL) @@ -462,6 +462,98 @@ GOOGLE_CLIENT_ID = '' GOOGLE_AUTH_ENDPOINT = '' GOOGLE_SECRET = '' +ONELOGIN_APP_ID = '' # OneLogin App ID provider by your administrator +ONELOGIN_EMAIL_FIELD = 'User.email' # SAML attribute used to provide email address +ONELOGIN_DEFAULT_ROLE = 'View' # Default RBAC when user doesn't already exist +ONELOGIN_HTTPS = True # If using HTTPS strict mode will check the requests are HTTPS +ONELOGIN_SETTINGS = { + # If strict is True, then the Python Toolkit will reject unsigned + # or unencrypted messages if it expects them to be signed or encrypted. + # Also it will reject the messages if the SAML standard is not strictly + # followed. Destination, NameId, Conditions ... are validated too. + "strict": True, + + # Enable debug mode (outputs errors). + "debug": True, + + # Service Provider Data that we are deploying. + "sp": { + # Identifier of the SP entity (must be a URI) + "entityId": "{BASE}metadata/".format(BASE=BASE_URL), + # Specifies info about where and how the message MUST be + # returned to the requester, in this case our SP. + "assertionConsumerService": { + # URL Location where the from the IdP will be returned + "url": "{BASE}api/1/auth/onelogin?acs".format(BASE=BASE_URL), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports this endpoint for the + # HTTP-POST binding only. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST" + }, + # If you need to specify requested attributes, set a + # attributeConsumingService. nameFormat, attributeValue and + # friendlyName can be omitted + #"attributeConsumingService": { + # "ServiceName": "SP test", + # "serviceDescription": "Test Service", + # "requestedAttributes": [ + # { + # "name": "", + # "isRequired": False, + # "nameFormat": "", + # "friendlyName": "", + # "attributeValue": "" + # } + # ] + #}, + # Specifies info about where and how the message MUST be + # returned to the requester, in this case our SP. + "singleLogoutService": { + # URL Location where the from the IdP will be returned + "url": "{BASE}api/1/auth/onelogin?sls".format(BASE=BASE_URL), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports the HTTP-Redirect binding + # only for this endpoint. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" + }, + # Specifies the constraints on the name identifier to be used to + # represent the requested subject. + # Take a look on src/onelogin/saml2/constants.py to see the NameIdFormat that are supported. + "NameIDFormat": "urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified", + # Usually x509cert and privateKey of the SP are provided by files placed at + # the certs folder. But we can also provide them with the following parameters + "x509cert": "", + "privateKey": "" + }, + + # Identity Provider Data that we want connected with our SP. + "idp": { + # Identifier of the IdP entity (must be a URI) + "entityId": "https://app.onelogin.com/saml/metadata/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), + # SSO endpoint info of the IdP. (Authentication Request protocol) + "singleSignOnService": { + # URL Target of the IdP where the Authentication Request Message + # will be sent. + "url": "https://app.onelogin.com/trust/saml2/http-post/sso/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports the HTTP-Redirect binding + # only for this endpoint. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" + }, + # SLO endpoint info of the IdP. + "singleLogoutService": { + # URL Location of the IdP where SLO Request will be sent. + "url": "https://app.onelogin.com/trust/saml2/http-redirect/slo/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports the HTTP-Redirect binding + # only for this endpoint. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" + }, + # Public x509 certificate of the IdP + "x509cert": "" + } +} + from datetime import timedelta PERMANENT_SESSION_LIFETIME=timedelta(minutes=60) # Will logout users after period of inactivity. SESSION_REFRESH_EACH_REQUEST=True diff --git a/scripts/secmonkey_role_setup.py b/scripts/secmonkey_role_setup.py index 1c55303cf..2b746c7a3 100755 --- a/scripts/secmonkey_role_setup.py +++ b/scripts/secmonkey_role_setup.py @@ -59,6 +59,7 @@ "acm:describecertificate", "acm:listcertificates", "cloudtrail:describetrails", + "cloudtrail:gettrailstatus", "config:describeconfigrules", "config:describeconfigurationrecorders", "directconnect:describeconnections", @@ -130,6 +131,7 @@ "route53:listhostedzones", "route53:listresourcerecordsets", "route53domains:listdomains", + "route53domains:getdomaindetail", "s3:getbucketacl", "s3:getbucketlocation", "s3:getbucketlogging", diff --git a/security_monkey/__init__.py b/security_monkey/__init__.py index 4fcf40aa3..a8053f0e6 100644 --- a/security_monkey/__init__.py +++ b/security_monkey/__init__.py @@ -160,6 +160,9 @@ def send_email(msg): api.add_resource(AuditorSettingsGet, '/api/1/auditorsettings') api.add_resource(AuditorSettingsPut, '/api/1/auditorsettings/') +from security_monkey.views.account_config import AccountConfigGet +api.add_resource(AccountConfigGet, '/api/1/account_config/') + ## Jira Sync import os from security_monkey.jirasync import JiraSync diff --git a/security_monkey/account_manager.py b/security_monkey/account_manager.py new file mode 100644 index 000000000..61516c786 --- /dev/null +++ b/security_monkey/account_manager.py @@ -0,0 +1,198 @@ +# Copyright 2016 Bridgewater Associates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +.. module: security_monkey.account + :platform: Unix + :synopsis: Base class for aws and other custom account types. + + +.. version:: $$VERSION$$ +.. moduleauthor:: Bridgewater OSS + + +""" +from datastore import Account, AccountType, AccountTypeCustomValues +from security_monkey import app, db +from security_monkey.common.utils import find_modules + +account_registry = {} + + +class AccountManagerType(type): + """ + Generates a global account regstry as AccountManager derived classes + are loaded + """ + def __init__(cls, name, bases, attrs): + super(AccountManagerType, cls).__init__(name, bases, attrs) + if cls.account_type: + app.logger.info("Registering account %s %s", + cls.account_type, cls.__name__) + account_registry[cls.account_type] = cls + + +class CustomFieldConfig(object): + """ + Defines additional field types for custom account types + """ + + def __init__(self, name, label, db_item, tool_tip, password=False): + super(CustomFieldConfig, self).__init__() + self.name = name + self.label = label + self.db_item = db_item + self.tool_tip = tool_tip + self.password = password + + +class AccountManager(object): + __metaclass__ = AccountManagerType + account_type = None + compatable_account_types = [] + custom_field_configs = [] + identifier_label = None + identifier_tool_tip = None + + def update(self, account_id, account_type, name, active, third_party, notes, + identifier, custom_fields=None): + """ + Updates an existing account in the database. + """ + account_type_result = _get_or_create_account_type(account_type) + query = Account.query.filter(Account.id == account_id) + if query.count(): + account = query.first() + else: + app.logger.info( + 'Account with id {} does not exist exists'.format(account_id)) + return None + + account = self._populate_account(account, account_type_result.id, name, + active, third_party, notes, identifier, custom_fields) + + db.session.add(account) + db.session.commit() + db.session.refresh(account) + account = self._load(account) + return account + + def create(self, account_type, name, active, third_party, notes, identifier, + custom_fields=None): + """ + Creates an account in the database. + """ + account_type_result = _get_or_create_account_type(account_type) + account = Account() + account = self._populate_account(account, account_type_result.id, name, + active, third_party, notes, identifier, custom_fields) + + db.session.add(account) + db.session.commit() + db.session.refresh(account) + account = self._load(account) + return account + + def lookup_account_by_identifier(self, identifier): + query = Account.query.filter(Account.identifier == identifier) + + if query.count(): + return query.first() + else: + return None + + def _load(self, account): + """ + Placeholder for additional load related processing to be implemented + by account type specific subclasses + """ + return account + + def _populate_account(self, account, account_type_id, name, active, third_party, + notes, identifier, custom_fields=None): + """ + Creates account DB object to be stored in the DB by create or update. + May be overridden to store additional data + """ + account.name = name + account.identifier = identifier + account.notes = notes + account.active = active + account.third_party = third_party + account.account_type_id = account_type_id + if account.custom_fields is None: + account.custom_fields = [] + + for custom_config in self.custom_field_configs: + if custom_config.db_item: + field_name = custom_config.name + for current_field in account.custom_fields: + if current_field.name == field_name: + if current_field.value != custom_fields.get(field_name): + current_field.value = custom_fields.get(field_name) + db.session.add(current_field) + break + else: + new_value = AccountTypeCustomValues( + name=field_name, value=custom_fields.get(field_name)) + account.custom_fields.append(new_value) + + return account + + def is_compatible_with_account_type(self, account_type): + if self.account_type == account_type or account_type in self.compatable_account_types: + return True + return False + + +def load_all_account_types(): + """ Verifies all account types are in the database """ + for account_type in account_registry.keys(): + _get_or_create_account_type(account_type) + + +def _get_or_create_account_type(account_type): + account_type_result = AccountType.query.filter( + AccountType.name == account_type).first() + if not account_type_result: + account_type_result = AccountType(name=account_type) + db.session.add(account_type_result) + db.session.commit() + app.logger.info("Creating a new AccountType: {} - ID: {}" + .format(account_type, account_type_result.id)) + + return account_type_result + + +def get_account_by_id(account_id): + """ + Retrieves an account plus any additional custom fields + """ + account = Account.query.filter(Account.id == account_id).first() + manager_class = account_registry.get(account.account_type.name) + account = manager_class()._load(account) + db.session.expunge(account) + return account + + +def get_account_by_name(account_name): + """ + Retrieves an account plus any additional custom fields + """ + account = Account.query.filter(Account.name == account_name).first() + manager_class = account_registry.get(account.account_type.name) + account = manager_class()._load(account) + db.session.expunge(account) + return account + +find_modules('account_managers') diff --git a/security_monkey/account_managers/__init__.py b/security_monkey/account_managers/__init__.py new file mode 100644 index 000000000..d6db0b8c6 --- /dev/null +++ b/security_monkey/account_managers/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2016 Bridgewater Associates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/security_monkey/account_managers/aws_account.py b/security_monkey/account_managers/aws_account.py new file mode 100644 index 000000000..720fc4942 --- /dev/null +++ b/security_monkey/account_managers/aws_account.py @@ -0,0 +1,72 @@ +# Copyright 2016 Bridgewater Associates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +.. module: security_monkey.accounts.aws_account + :platform: Unix + :synopsis: Manages generic AWS account. + + +.. version:: $$VERSION$$ +.. moduleauthor:: Bridgewater OSS + + +""" +from security_monkey.account_manager import AccountManager, CustomFieldConfig +from security_monkey.datastore import Account + + +class AWSAccountManager(AccountManager): + account_type = 'AWS' + identifier_label = 'Number' + identifier_tool_tip = 'Enter the AWS account number, if you have it. (12 digits)' + s3_name_label = ('The S3 Name is the way AWS presents the account in an ACL policy. ' + 'This is often times the first part of the email address that was used ' + 'to create the Amazon account. (myaccount@example.com may be represented ' + 'as myaccount\). If you see S3 issues appear for unknown cross account ' + 'access, you may need to update the S3 Name.') + role_name_label = ("Optional custom role name, otherwise the default 'SecurityMonkey' is used. " + "When deploying roles via CloudFormation, this is the Physical ID of the generated IAM::ROLE.") + custom_field_configs = [ + CustomFieldConfig('s3_name', 'S3 Name', True, s3_name_label), + CustomFieldConfig('role_name', 'Role Name', True, role_name_label) + ] + + def __init__(self): + super(AWSAccountManager, self).__init__() + + def lookup_account_by_identifier(self, identifier): + """ + Overrides the lookup to also check the number for backwards compatibility + """ + account = super(AWSAccountManager, + self).lookup_account_by_identifier(identifier) + if account is None: + query = Account.query.filter(Account.number == identifier) + if query.count(): + account = query.first() + return account + + def _populate_account(self, account, account_type, name, active, third_party, + notes, identifier, custom_fields=None): + """ + Overrides create and update to also save the number, s3_name and role_name + for backwards compatibility + """ + account = super(AWSAccountManager, self)._populate_account(account, account_type, name, active, third_party, + notes, identifier, custom_fields) + + account.number = identifier + account.s3_name = account.getCustom("s3_name") + account.role_name = account.getCustom("role_name") + return account diff --git a/security_monkey/account_managers/custom/__init__.py b/security_monkey/account_managers/custom/__init__.py new file mode 100644 index 000000000..d6db0b8c6 --- /dev/null +++ b/security_monkey/account_managers/custom/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2016 Bridgewater Associates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/security_monkey/account_managers/custom/sample_active_directory.py b/security_monkey/account_managers/custom/sample_active_directory.py new file mode 100644 index 000000000..d9004b290 --- /dev/null +++ b/security_monkey/account_managers/custom/sample_active_directory.py @@ -0,0 +1,39 @@ +# Copyright 2016 Bridgewater Associates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +.. module: security_monkey.accounts.sample_active_directory + :platform: Unix + :synopsis: Manages Active Directory account. Adds fields needed to connect + as custom fields + +.. version:: $$VERSION$$ +.. moduleauthor:: Bridgewater OSS + + +""" +# from security_monkey.account_manager import AccountManager, CustomFieldConfig +# +# class ActiveDirectoryAccountManager(AccountManager): +# account_type = 'ACTIVE_DIRECTORY' +# identifier_label = 'URL' +# identifier_tool_tip = 'Enter the URL of the active directory account' +# custom_field_configs = [ +# CustomFieldConfig('user_id', 'User ID', True, +# 'Enter the user id for the account'), +# CustomFieldConfig('password', 'Password', True, +# "Enter the user's password to access the account", True) +# ] +# +# def __init__(self): +# super(AccountManager, self).__init__() diff --git a/security_monkey/account_managers/custom/sample_db_extended_aws.py b/security_monkey/account_managers/custom/sample_db_extended_aws.py new file mode 100644 index 000000000..60e851144 --- /dev/null +++ b/security_monkey/account_managers/custom/sample_db_extended_aws.py @@ -0,0 +1,41 @@ +# Copyright 2016 Bridgewater Associates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +.. module: security_monkey.accounts.sample_extended_aws + :platform: Unix + :synopsis: Extends an AWS account with additional qualifiers that may be + used to in the applied_to_account method in custom auditors + + +.. version:: $$VERSION$$ +.. moduleauthor:: Bridgewater OSS + + +""" +# from security_monkey.account_manager import CustomFieldConfig +# from security_monkey.account_managers.aws_account import AWSAccountManager +# from security_monkey.datastore import Account, AccountTypeCustomValues +# from security_monkey import app +# +# +# class DBExtendedAWSAccountManager(AWSAccountManager): +# account_type = 'DB_EXTENDED_AWS' +# compatable_account_types = ['AWS'] +# custom_field_configs = AWSAccountManager.custom_field_configs + [ +# CustomFieldConfig('security_level', 'Security Level', True, +# 'A numeric value used to indicated the risk'), +# ] +# +# def __init__(self): +# super(ExtendedAWSAccountManager, self).__init__() diff --git a/security_monkey/account_managers/custom/sample_extended_aws.py b/security_monkey/account_managers/custom/sample_extended_aws.py new file mode 100644 index 000000000..adf2a69c7 --- /dev/null +++ b/security_monkey/account_managers/custom/sample_extended_aws.py @@ -0,0 +1,55 @@ +# Copyright 2016 Bridgewater Associates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +.. module: security_monkey.accounts.sample_extended_aws + :platform: Unix + :synopsis: Extends an AWS account with additional qualifiers that may be + used to in the applied_to_account method in custom auditors + + +.. version:: $$VERSION$$ +.. moduleauthor:: Bridgewater OSS + + +""" +# from security_monkey.account_manager import CustomFieldConfig +# from security_monkey.account_managers.aws_account import AWSAccountManager +# from security_monkey.datastore import Account, AccountTypeCustomValues +# from security_monkey import app +# +# +# class ExtendedAWSAccountManager(AWSAccountManager): +# account_type = 'EXTENDED_AWS' +# compatable_account_types = ['AWS'] +# custom_field_configs = AWSAccountManager.custom_field_configs + [ +# CustomFieldConfig('security_level', 'Security Level', False, +# 'A numeric value used to indicated the risk'), +# ] +# +# def __init__(self): +# super(ExtendedAWSAccountManager, self).__init__() +# +# def _load(self, account): +# """ +# Demonstrates ability to retrieve data from some other system of record +# """ +# account.custom_fields.append(AccountTypeCustomValues(name='security_level', +# value=self._getFromConfig(account.identifier))) +# return account +# +# def _getFromConfig(self, account_number): +# """ +# Currently hard coded but could access some other datasource +# """ +# return "5" diff --git a/security_monkey/auditor.py b/security_monkey/auditor.py index e5e4c0def..cd43b32ec 100644 --- a/security_monkey/auditor.py +++ b/security_monkey/auditor.py @@ -59,6 +59,8 @@ class Auditor(object): i_am_singular = None # Should be overridden i_am_plural = None # Should be overridden __metaclass__ = AuditorType + support_auditor_indexes = [] + support_watcher_indexes = [] def __init__(self, accounts=None, debug=False): self.datastore = datastore.Datastore() @@ -67,6 +69,7 @@ def __init__(self, accounts=None, debug=False): self.items = [] self.team_emails = app.config.get('SECURITY_TEAM_EMAIL', []) self.emails = [] + self.current_support_items = {} if type(self.team_emails) in (str, unicode): self.emails.append(self.team_emails) @@ -85,8 +88,8 @@ def add_issue(self, score, issue, item, notes=None): :return: The new issue """ - if notes and len(notes) > 512: - notes = notes[0:512] + if notes and len(notes) > 1024: + notes = notes[0:1024] for existing_issue in item.audit_issues: if existing_issue.issue == issue: @@ -123,6 +126,8 @@ def audit_these_objects(self, items): """ app.logger.debug("Asked to audit {} Objects".format(len(items))) self.prep_for_audit() + self.current_support_items = {} + methods = [getattr(self, method_name) for method_name in dir(self) if method_name.find("check_") == 0] app.logger.debug("methods: {}".format(methods)) for item in items: @@ -152,19 +157,48 @@ def read_previous_items(self): region=item.region, account=item.account.name, name=item.name, + arn=item.arn, new_config=item_revision.config) new_item.audit_issues = [] new_item.db_item = item prev_list.append(new_item) return prev_list + def read_previous_items_for_account(self, index, account): + """ + Pulls the last-recorded configuration from the database. + :return: List of all items for the given technology and the given account. + """ + prev_list = [] + prev = self.datastore.get_all_ctype_filtered(tech=index, account=account, include_inactive=False) + # Returns a map of {Item: ItemRevision} + for item in prev: + item_revision = prev[item] + new_item = ChangeItem(index=self.index, + region=item.region, + account=item.account.name, + name=item.name, + arn=item.arn, + new_config=item_revision.config) + new_item.audit_issues = [] + new_item.db_item = item + prev_list.append(new_item) + + return prev_list + def save_issues(self): """ Save all new issues. Delete all fixed issues. """ app.logger.debug("\n\nSaving Issues.") + + # Work around for issue where previous get's may cause commit to fail + db.session.rollback() for item in self.items: + changes = False + loaded = False if not hasattr(item, 'db_item'): + loaded = True item.db_item = self.datastore._get_item(item.index, item.region, item.account, item.name) existing_issues = list(item.db_item.issues) @@ -175,25 +209,26 @@ def save_issues(self): self._set_auditor_setting_for_issue(issue) # Add new issues - old_scored = ["{} -- {} -- {} -- {}".format( + old_scored = ["{} -- {} -- {} -- {} -- {}".format( old_issue.auditor_setting.auditor_class, old_issue.issue, old_issue.notes, - old_issue.score) for old_issue in existing_issues] + old_issue.score, + self._item_list_string(old_issue)) for old_issue in existing_issues] for new_issue in new_issues: - nk = "{} -- {} -- {} -- {}".format(self.__class__.__name__, + nk = "{} -- {} -- {} -- {} -- {}".format(self.__class__.__name__, new_issue.issue, new_issue.notes, - new_issue.score) + new_issue.score, + self._item_list_string(new_issue)) if nk not in old_scored: + changes = True app.logger.debug("Saving NEW issue {}".format(nk)) item.found_new_issue = True item.confirmed_new_issues.append(new_issue) item.db_item.issues.append(new_issue) - db.session.add(item.db_item) - db.session.add(new_issue) else: for issue in existing_issues: if issue.issue == new_issue.issue and issue.notes == new_issue.notes and issue.score == new_issue.score: @@ -203,20 +238,29 @@ def save_issues(self): app.logger.debug("Issue was previously found. Not overwriting.\n\t{}\n\t{}".format(key, nk)) # Delete old issues - new_scored = ["{} -- {} -- {}".format(new_issue.issue, + new_scored = ["{} -- {} -- {} -- {}".format(new_issue.issue, new_issue.notes, - new_issue.score) for new_issue in new_issues] + new_issue.score, + self._item_list_string(new_issue)) for new_issue in new_issues] for old_issue in existing_issues: - ok = "{} -- {} -- {}".format(old_issue.issue, + ok = "{} -- {} -- {} -- {}".format(old_issue.issue, old_issue.notes, - old_issue.score) + old_issue.score, + self._item_list_string(old_issue)) old_issue_class = old_issue.auditor_setting.auditor_class if old_issue_class is None or (old_issue_class == self.__class__.__name__ and ok not in new_scored): - app.logger.debug("Deleting FIXED issue {}".format(ok)) + changes = True + app.logger.debug("Deleting FIXED or REPLACED issue {}".format(ok)) item.confirmed_fixed_issues.append(old_issue) - db.session.delete(old_issue) + item.db_item.issues.remove(old_issue) + + if changes: + db.session.add(item.db_item) + else: + if loaded: + db.session.expunge(item.db_item) db.session.commit() self._create_auditor_settings() @@ -258,6 +302,13 @@ def create_report(self): else: return False + def applies_to_account(self, account): + """ + Placeholder for custom auditors which may only want to run against + certain types of accounts + """ + return True + def _create_auditor_settings(self): """ Checks to see if an AuditorSettings entry exists for each issue. @@ -352,3 +403,91 @@ def _check_cross_account_root(self, source_item, dest_arn, actions): .format(dest_arn.account_number) notes += "{}".format(actions) self.add_issue(6, tag, source_item, notes=notes) + + def get_auditor_support_items(self, auditor_index, account): + for index in self.support_auditor_indexes: + if index == auditor_index: + audited_items = self.current_support_items.get(account + auditor_index) + if audited_items is None: + audited_items = self.read_previous_items_for_account(auditor_index, account) + if not audited_items: + app.logger.info("{} Could not load audited items for {}/{}".format(self.index, auditor_index, account)) + self.current_support_items[account+auditor_index] = [] + else: + self.current_support_items[account+auditor_index] = audited_items + return audited_items + + raise Exception("Auditor {} is not configured as an audit support auditor for {}".format(auditor_index, self.index)) + + def get_watcher_support_items(self, watcher_index, account): + for index in self.support_watcher_indexes: + if index == watcher_index: + items = self.current_support_items.get(account + watcher_index) + if items is None: + items = self.read_previous_items_for_account(watcher_index, account) + # Only the item contents should be used for watcher support + # config. This prevents potentially stale issues from being + # used by the auditor + for item in items: + item.db_item.issues = [] + + if not items: + app.logger.info("{} Could not load support items for {}/{}".format(self.index, watcher_index, account)) + self.current_support_items[account+watcher_index] = [] + else: + self.current_support_items[account+watcher_index] = items + return items + + raise Exception("Watcher {} is not configured as a data support watcher for {}".format(watcher_index, self.index)) + + def link_to_support_item_issues(self, item, sub_item, sub_issue_message=None, issue_message=None, issue=None, score=None): + """ + Creates a new issue that is linked to an issue in a support auditor + """ + matching_issues = [] + for sub_issue in sub_item.issues: + if not sub_issue_message or sub_issue.issue == sub_issue_message: + matching_issues.append(sub_issue) + + if len(matching_issues) > 0: + for matching_issue in matching_issues: + if issue is None: + if issue_message is None: + if sub_issue_message is not None: + issue_message = sub_issue_message + else: + issue_message = "UNDEFINED" + + if score is not None: + issue = self.add_issue(score, issue_message, item) + else: + issue = self.add_issue(matching_issue.score, issue_message, item) + else: + if score is not None: + issue.score = score + else: + issue.score = issue.score + matching_issue.score + + issue.sub_items.append(sub_item) + + return issue + + def link_to_support_item(self, score, issue_message, item, sub_item, issue=None): + """ + Creates a new issue that is linked a support watcher item + """ + if issue is None: + issue = self.add_issue(score, issue_message, item) + issue.sub_items.append(sub_item) + return issue + + def _item_list_string(self, issue): + """ + Use by save_issue to generate a unique id for an item + """ + item_ids = [] + for sub_item in issue.sub_items: + item_ids.append(sub_item.id) + + item_ids.sort() + return str(item_ids) diff --git a/security_monkey/auditors/cloudtrail.py b/security_monkey/auditors/cloudtrail.py new file mode 100644 index 000000000..d6a821d76 --- /dev/null +++ b/security_monkey/auditors/cloudtrail.py @@ -0,0 +1,44 @@ +# Copyright 2016 Netflix, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +.. module: security_monkey.auditors.cloudtrail + :platform: Unix + +.. version:: $$VERSION$$ +.. moduleauthor:: Nag Medida + +""" +from security_monkey.auditor import Auditor +from security_monkey.watchers.cloud_trail import CloudTrail + + +class CloudTrailAuditor(Auditor): + index = CloudTrail.index + i_am_singular = CloudTrail.i_am_singular + i_am_plural = CloudTrail.i_am_plural + + def __init__(self, accounts=None, debug=False): + super(CloudTrailAuditor, self).__init__(accounts=accounts, debug=debug) + + def check_if_cloudtrail_in_all_regions(self, cloud_trail): + if not cloud_trail.config.get('is_multi_region_trail'): + message = "POLICY - CloudTrail is not enabled for multi-region" + self.add_issue(10, message, cloud_trail) + return False + + def check_if_cloudtrail_is_enabled(self, cloud_trail): + if not cloud_trail.config.get('trail_status'): + message = "POLICY - CloudTrail is disabled" + self.add_issue(10, message, cloud_trail) + return False diff --git a/security_monkey/auditors/elasticsearch_service.py b/security_monkey/auditors/elasticsearch_service.py index d5551472d..f09df3615 100644 --- a/security_monkey/auditors/elasticsearch_service.py +++ b/security_monkey/auditors/elasticsearch_service.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -.. module: security_monkey.auditors.elb +.. module: security_monkey.auditors.elasticsearch_service :platform: Unix .. version:: $$VERSION$$ @@ -51,13 +51,7 @@ def _parse_arn(self, arn_input, account_numbers, es_domain): self.add_issue(3, 'Auditor could not parse ARN', es_domain, notes=arn_input) return - if arn.tech == 's3': - notes = "The ElasticSearch Service domain allows access from S3 bucket [{}]. ".format(arn.name) - notes += "Security Monkey does not yet have the capability to determine if this is " - notes += "a friendly S3 bucket. Please verify manually." - self.add_issue(3, 'ES cluster allows access from S3 bucket', es_domain, notes=notes) - else: - account_numbers.append(arn.account_number) + account_numbers.append(arn.account_number) def check_es_access_policy(self, es_domain): policy = es_domain.config["policy"] diff --git a/security_monkey/auditors/iam/iam_group.py b/security_monkey/auditors/iam/iam_group.py index 989b91e5a..be035cbd2 100644 --- a/security_monkey/auditors/iam/iam_group.py +++ b/security_monkey/auditors/iam/iam_group.py @@ -21,12 +21,13 @@ """ from security_monkey.watchers.iam.iam_group import IAMGroup from security_monkey.auditors.iam.iam_policy import IAMPolicyAuditor - +from security_monkey.watchers.iam.managed_policy import ManagedPolicy class IAMGroupAuditor(IAMPolicyAuditor): index = IAMGroup.index i_am_singular = IAMGroup.i_am_singular i_am_plural = IAMGroup.i_am_plural + support_auditor_indexes = [ManagedPolicy.index] def __init__(self, accounts=None, debug=False): super(IAMGroupAuditor, self).__init__(accounts=accounts, debug=debug) @@ -69,3 +70,9 @@ def check_security_group_permissions(self, iamgroup_item): alert when an IAM Group has ec2:AuthorizeSecurityGroupEgress or ec2:AuthorizeSecurityGroupIngress. """ self.library_check_iamobj_has_security_group_permissions(iamgroup_item, policies_key='grouppolicies') + + def check_attached_managed_policies(self, iamgroup_item): + """ + alert when an IAM Group is attached to a managed policy with issues + """ + self.library_check_attached_managed_policies(iamgroup_item, 'group') diff --git a/security_monkey/auditors/iam/iam_policy.py b/security_monkey/auditors/iam/iam_policy.py index a3d462b35..dc8e81ea3 100644 --- a/security_monkey/auditors/iam/iam_policy.py +++ b/security_monkey/auditors/iam/iam_policy.py @@ -20,7 +20,8 @@ """ from security_monkey.auditor import Auditor - +from security_monkey.watchers.iam.managed_policy import ManagedPolicy +from security_monkey import app import json @@ -61,7 +62,7 @@ class IAMPolicyAuditor(Auditor): def __init__(self, accounts=None, debug=False): super(IAMPolicyAuditor, self).__init__(accounts=accounts, debug=debug) - def library_check_iamobj_has_star_privileges(self, iamobj_item, policies_key='userpolicies', multiple_policies=True): + def library_check_iamobj_has_star_privileges(self, iamobj_item, policies_key='InlinePolicies', multiple_policies=True): """ alert when an IAM Object has a policy allowing '*'. """ @@ -82,7 +83,7 @@ def check_statement(statement): else: _iterate_over_statements(iamobj_item.config[policies_key], check_statement) - def library_check_iamobj_has_iam_star_privileges(self, iamobj_item, policies_key='userpolicies', multiple_policies=True): + def library_check_iamobj_has_iam_star_privileges(self, iamobj_item, policies_key='InlinePolicies', multiple_policies=True): """ alert when an IAM Object has a policy allowing 'iam:*'. """ @@ -103,7 +104,7 @@ def check_statement(statement): else: _iterate_over_statements(iamobj_item.config[policies_key], check_statement) - def library_check_iamobj_has_iam_privileges(self, iamobj_item, policies_key='userpolicies', multiple_policies=True): + def library_check_iamobj_has_iam_privileges(self, iamobj_item, policies_key='InlinePolicies', multiple_policies=True): """ alert when an IAM Object has a policy allowing 'iam:XxxxxXxxx'. """ @@ -124,7 +125,7 @@ def check_statement(statement): else: _iterate_over_statements(iamobj_item.config[policies_key], check_statement) - def library_check_iamobj_has_iam_passrole(self, iamobj_item, policies_key='userpolicies', multiple_policies=True): + def library_check_iamobj_has_iam_passrole(self, iamobj_item, policies_key='InlinePolicies', multiple_policies=True): """ alert when an IAM Object has a policy allowing 'iam:PassRole'. This allows the object to pass any role specified in the resource block to an ec2 instance. @@ -146,7 +147,7 @@ def check_statement(statement): else: _iterate_over_statements(iamobj_item.config[policies_key], check_statement) - def library_check_iamobj_has_notaction(self, iamobj_item, policies_key='userpolicies', multiple_policies=True): + def library_check_iamobj_has_notaction(self, iamobj_item, policies_key='InlinePolicies', multiple_policies=True): """ alert when an IAM Object has a policy containing 'NotAction'. NotAction combined with an "Effect": "Allow" often provides more privilege @@ -164,7 +165,7 @@ def check_statement(statement): else: _iterate_over_statements(iamobj_item.config[policies_key], check_statement) - def library_check_iamobj_has_security_group_permissions(self, iamobj_item, policies_key='userpolicies', multiple_policies=True): + def library_check_iamobj_has_security_group_permissions(self, iamobj_item, policies_key='InlinePolicies', multiple_policies=True): """ alert when an IAM Object has ec2:AuthorizeSecurityGroupEgress or ec2:AuthorizeSecurityGroupIngress. """ @@ -186,3 +187,21 @@ def check_statement(statement): _iterate_over_sub_policies(iamobj_item.config.get(policies_key, {}), check_statement) else: _iterate_over_statements(iamobj_item.config[policies_key], check_statement) + + def library_check_attached_managed_policies(self, iam_item, iam_type): + """ + alert when an IAM item (group, user or role) is attached to a managed policy with issues + """ + mp_items = self.get_auditor_support_items(ManagedPolicy.index, iam_item.account) + managed_policies = iam_item.config.get('managed_policies', iam_item.config.get('ManagedPolicies')) + for item_mp in managed_policies or []: + found = False + item_mp_arn = item_mp.get('arn', item_mp.get('Arn')) + for mp_item in mp_items or [] and not found: + mp_arn = mp_item.config.get('arn', mp_item.config.get('Arn')) + if mp_arn == item_mp_arn: + found = True + self.link_to_support_item_issues(iam_item, mp_item.db_item, None, "Found issue(s) in attached Managed Policy") + + if not found: + app.logger.error("IAM Managed Policy defined but not found for {}-{}".format(iam_item.index, iam_item.name)) diff --git a/security_monkey/auditors/iam/iam_role.py b/security_monkey/auditors/iam/iam_role.py index ace762bc4..39821ee10 100644 --- a/security_monkey/auditors/iam/iam_role.py +++ b/security_monkey/auditors/iam/iam_role.py @@ -23,12 +23,15 @@ from security_monkey.watchers.iam.iam_role import IAMRole from security_monkey.auditors.iam.iam_policy import IAMPolicyAuditor +from security_monkey.watchers.iam.managed_policy import ManagedPolicy +from security_monkey.datastore import Account class IAMRoleAuditor(IAMPolicyAuditor): index = IAMRole.index i_am_singular = IAMRole.i_am_singular i_am_plural = IAMRole.i_am_plural + support_auditor_indexes = [ManagedPolicy.index] def __init__(self, accounts=None, debug=False): super(IAMRoleAuditor, self).__init__(accounts=accounts, debug=debug) @@ -53,44 +56,87 @@ def check_statement(statement): if aws and aws == "*": self.add_issue(10, tag, iamrole_item, notes=json.dumps(statement)) - elif aws and type(aws) is list: - for entry in aws: - if entry == "*": - self.add_issue(10, tag, iamrole_item, - notes=json.dumps(statement)) + elif aws and type(aws) is list: + for entry in aws: + if entry == "*": + self.add_issue(10, tag, iamrole_item, + notes=json.dumps(statement)) - assume_role_policy = iamrole_item.config.get("assume_role_policy_document", {}) + assume_role_policy = iamrole_item.config.get("AssumeRolePolicyDocument", {}) statement = assume_role_policy.get("Statement", []) - if type(statement) is list: - for single_statement in statement: - check_statement(single_statement) - elif type(statement) is dict: - check_statement(statement) + if type(statement) is dict: + statement = [statement] + for single_statement in statement: + check_statement(single_statement) + + def check_assume_role_from_unknown_account(self, iamrole_item): + """ + alert when an IAM Role has an assume_role_policy_document granting access to an unknown account + """ + + def check_statement(statement): + + def check_account_in_arn(input): + from security_monkey.common.arn import ARN + arn = ARN(input) + + if arn.error: + print('Could not parse ARN in Trust Policy: {arn}'.format(arn=input)) + + if not arn.error and arn.account_number: + account = Account.query.filter(Account.number == arn.account_number).first() + if not account: + tag = "IAM Role allows assume-role from an " \ + + "Unknown Account ({account_number})".format( + account_number=arn.account_number) + self.add_issue(10, tag, iamrole_item, notes=json.dumps(statement)) + + action = statement.get("Action", None) + if action and action == "sts:AssumeRole": + effect = statement.get("Effect", None) + if effect and effect == "Allow": + principal = statement.get("Principal", None) + if not principal: + return + if type(principal) is dict: + aws = principal.get("AWS", None) + if aws and type(aws) is list: + for arn in aws: + check_account_in_arn(arn) + elif aws: + check_account_in_arn(aws) + + assume_role_policy = iamrole_item.config.get("AssumeRolePolicyDocument", {}) + statement = assume_role_policy.get("Statement", []) + if type(statement) is dict: + statement = [statement] + for single_statement in statement: + check_statement(single_statement) def check_star_privileges(self, iamrole_item): """ alert when an IAM Role has a policy allowing '*'. """ - self.library_check_iamobj_has_star_privileges(iamrole_item, policies_key='rolepolicies') + self.library_check_iamobj_has_star_privileges(iamrole_item, policies_key='InlinePolicies') def check_iam_star_privileges(self, iamrole_item): """ alert when an IAM Role has a policy allowing 'iam:*'. """ - self.library_check_iamobj_has_iam_star_privileges(iamrole_item, policies_key='rolepolicies') + self.library_check_iamobj_has_iam_star_privileges(iamrole_item, policies_key='InlinePolicies') def check_iam_privileges(self, iamrole_item): """ alert when an IAM Role has a policy allowing 'iam:XxxxxXxxx'. """ - self.library_check_iamobj_has_iam_privileges(iamrole_item, policies_key='rolepolicies') + self.library_check_iamobj_has_iam_privileges(iamrole_item, policies_key='InlinePolicies') def check_iam_passrole(self, iamrole_item): """ alert when an IAM Role has a policy allowing 'iam:PassRole'. This allows the role to pass any role specified in the resource block to an ec2 instance. """ - self.library_check_iamobj_has_iam_passrole(iamrole_item, policies_key='rolepolicies') + self.library_check_iamobj_has_iam_passrole(iamrole_item, policies_key='InlinePolicies') def check_notaction(self, iamrole_item): """ @@ -98,10 +144,16 @@ def check_notaction(self, iamrole_item): NotAction combined with an "Effect": "Allow" often provides more privilege than is desired. """ - self.library_check_iamobj_has_notaction(iamrole_item, policies_key='rolepolicies') + self.library_check_iamobj_has_notaction(iamrole_item, policies_key='InlinePolicies') def check_security_group_permissions(self, iamrole_item): """ alert when an IAM Role has ec2:AuthorizeSecurityGroupEgress or ec2:AuthorizeSecurityGroupIngress. """ - self.library_check_iamobj_has_security_group_permissions(iamrole_item, policies_key='rolepolicies') + self.library_check_iamobj_has_security_group_permissions(iamrole_item, policies_key='InlinePolicies') + + def check_attached_managed_policies(self, iamrole_item): + """ + alert when an IAM Role is attached to a managed policy with issues + """ + self.library_check_attached_managed_policies(iamrole_item, 'role') diff --git a/security_monkey/auditors/iam/iam_user.py b/security_monkey/auditors/iam/iam_user.py index 794f309bd..d3a0ea9cd 100644 --- a/security_monkey/auditors/iam/iam_user.py +++ b/security_monkey/auditors/iam/iam_user.py @@ -26,12 +26,13 @@ from security_monkey.watchers.iam.iam_user import IAMUser from security_monkey.auditors.iam.iam_policy import IAMPolicyAuditor - +from security_monkey.watchers.iam.managed_policy import ManagedPolicy class IAMUserAuditor(IAMPolicyAuditor): index = IAMUser.index i_am_singular = IAMUser.i_am_singular i_am_plural = IAMUser.i_am_plural + support_auditor_indexes = [ManagedPolicy.index] def __init__(self, accounts=None, debug=False): super(IAMUserAuditor, self).__init__(accounts=accounts, debug=debug) @@ -49,23 +50,23 @@ def check_access_keys(self, iamuser_item): """ alert when an IAM User has an active access key. """ - akeys = iamuser_item.config.get('accesskeys', {}) - for akey in akeys.keys(): - if u'status' in akeys[akey]: - if akeys[akey][u'status'] == u'Active': - self.add_issue(1, 'User has active accesskey.', iamuser_item, notes=akey) + akeys = iamuser_item.config.get('AccessKeys', {}) + for akey in akeys: + if 'Status' in akey: + if akey['Status'] == 'Active': + self.add_issue(1, 'User has active accesskey.', iamuser_item, notes=akey['AccessKeyId']) else: - self.add_issue(0, 'User has an inactive accesskey.', iamuser_item, notes=akey) + self.add_issue(0, 'User has an inactive accesskey.', iamuser_item, notes=akey['AccessKeyId']) def check_access_key_rotation(self, iamuser_item): """ alert when an IAM User has an active access key created more than 90 days go. """ - akeys = iamuser_item.config.get('accesskeys', {}) - for akey in akeys.keys(): - if u'status' in akeys[akey]: - if akeys[akey][u'status'] == u'Active': - create_date = akeys[akey][u'create_date'] + akeys = iamuser_item.config.get('AccessKeys', {}) + for akey in akeys: + if 'Status' in akey: + if akey['Status'] == 'Active': + create_date = akey['CreateDate'] create_date = parser.parse(create_date) if create_date < self.ninety_days_ago: notes = "> 90 days ago" @@ -75,11 +76,11 @@ def check_access_key_last_used(self, iamuser_item): """ alert if an active access key hasn't been used in 90 days """ - akeys = iamuser_item.config.get('accesskeys', {}) - for akey in akeys.keys(): - if u'status' in akeys[akey]: - if akeys[akey][u'status'] == u'Active': - last_used_str = akeys[akey].get(u'LastUsedDate') + akeys = iamuser_item.config.get('AccessKeys', {}) + for akey in akeys: + if 'Status' in akey: + if akey['Status'] == 'Active': + last_used_str = akey.get('LastUsedDate') if not last_used_str: continue last_used_date = parser.parse(last_used_str) @@ -91,26 +92,26 @@ def check_star_privileges(self, iamuser_item): """ alert when an IAM User has a policy allowing '*'. """ - self.library_check_iamobj_has_star_privileges(iamuser_item, policies_key='userpolicies') + self.library_check_iamobj_has_star_privileges(iamuser_item, policies_key='InlinePolicies') def check_iam_star_privileges(self, iamuser_item): """ alert when an IAM User has a policy allowing 'iam:*'. """ - self.library_check_iamobj_has_iam_star_privileges(iamuser_item, policies_key='userpolicies') + self.library_check_iamobj_has_iam_star_privileges(iamuser_item, policies_key='InlinePolicies') def check_iam_privileges(self, iamuser_item): """ alert when an IAM User has a policy allowing 'iam:XxxxxXxxx'. """ - self.library_check_iamobj_has_iam_privileges(iamuser_item, policies_key='userpolicies') + self.library_check_iamobj_has_iam_privileges(iamuser_item, policies_key='InlinePolicies') def check_iam_passrole(self, iamuser_item): """ alert when an IAM User has a policy allowing 'iam:PassRole'. This allows the user to pass any role specified in the resource block to an ec2 instance. """ - self.library_check_iamobj_has_iam_passrole(iamuser_item, policies_key='userpolicies') + self.library_check_iamobj_has_iam_passrole(iamuser_item, policies_key='InlinePolicies') def check_notaction(self, iamuser_item): """ @@ -118,28 +119,22 @@ def check_notaction(self, iamuser_item): NotAction combined with an "Effect": "Allow" often provides more privilege than is desired. """ - self.library_check_iamobj_has_notaction(iamuser_item, policies_key='userpolicies') + self.library_check_iamobj_has_notaction(iamuser_item, policies_key='InlinePolicies') def check_security_group_permissions(self, iamuser_item): """ alert when an IAM User has ec2:AuthorizeSecurityGroupEgress or ec2:AuthorizeSecurityGroupIngress. """ - self.library_check_iamobj_has_security_group_permissions(iamuser_item, policies_key='userpolicies') + self.library_check_iamobj_has_security_group_permissions(iamuser_item, policies_key='InlinePolicies') def check_no_mfa(self, iamuser_item): """ alert when an IAM user has a login profile and no MFA devices. This means a human account which could be better protected with 2FA. """ - mfas = iamuser_item.config.get('mfadevices', {}) - loginprof = iamuser_item.config.get('loginprofile', {}) - has_active_mfas = False - has_login_profile = False - if mfas: - has_active_mfas = True - if loginprof != {}: - has_login_profile = True - if has_login_profile and not has_active_mfas: + user_mfas = iamuser_item.config.get('MfaDevices', {}) + login_profile = iamuser_item.config.get('LoginProfile', {}) + if login_profile and not user_mfas: self.add_issue(1, 'User with password login and no MFA devices.', iamuser_item) def check_loginprofile_plus_akeys(self, iamuser_item): @@ -147,11 +142,17 @@ def check_loginprofile_plus_akeys(self, iamuser_item): alert when an IAM user has a login profile and API access via access keys. An account should be used Either for API access OR for console access, but maybe not both. """ - if not iamuser_item.config.get('loginprofile', None): + if not iamuser_item.config.get('LoginProfile', None): return - akeys = iamuser_item.config.get('accesskeys', {}) - for akey in akeys.keys(): - if u'status' in akeys[akey] and akeys[akey][u'status'] == u'Active': + akeys = iamuser_item.config.get('AccessKeys', {}) + for akey in akeys: + if 'Status' in akey and akey['Status'] == 'Active': self.add_issue(1, 'User with password login and API access.', iamuser_item) return + + def check_attached_managed_policies(self, iamuser_item): + """ + alert when an IAM Role is attached to a managed policy with issues + """ + self.library_check_attached_managed_policies(iamuser_item, 'user') diff --git a/security_monkey/auditors/kms.py b/security_monkey/auditors/kms.py index 936d9e780..e11a72ab1 100644 --- a/security_monkey/auditors/kms.py +++ b/security_monkey/auditors/kms.py @@ -22,6 +22,25 @@ import json +def extract_condition_account_numbers(condition): + condition_subsection = condition.get('StringLike', {}) or \ + condition.get('ForAllValues:StringLike', {}) or \ + condition.get('ForAnyValue:StringLike', {}) or \ + condition.get('StringEquals', {}) or \ + condition.get('ForAllValues:StringEquals', {}) or \ + condition.get('ForAnyValue:StringEquals', {}) + + condition_accounts = [] + for key, value in condition_subsection.iteritems(): + if key.lower() == 'kms:calleraccount': + if isinstance(value, list): + condition_accounts.extend(value) + else: + condition_accounts.append(value) + + return condition_accounts + + class KMSAuditor(Auditor): index = KMS.index i_am_singular = KMS.i_am_singular @@ -39,26 +58,38 @@ def check_for_kms_policy_with_foreign_account(self, kms_item): key_account_id = kms_item.config.get("AWSAccountId") key_policies = kms_item.config.get("Policies") - has_issue = False - bad_statements = [] - for policy in key_policies: for statement in policy.get("Statement"): + if 'Condition' in statement: + condition = statement.get('Condition') + condition_accounts = [] + if condition: + condition_accounts = extract_condition_account_numbers(condition) + + cross_accounts = [account for account in condition_accounts if account != key_account_id] + if cross_accounts: + notes = "Condition - kms:CallerAccount: {}".format(json.dumps(cross_accounts)) + self.add_issue(5, tag, kms_item, notes=notes) + if statement and statement.get("Principal"): - if isinstance(statement.get("Principal"), basestring): + aws_principal = statement.get("Principal") + if isinstance(aws_principal, dict): + if 'AWS' in aws_principal: + aws_principal = aws_principal.get("AWS") + elif 'Service' in aws_principal: + aws_principal = aws_principal.get("Service") + + if isinstance(aws_principal, basestring): # Handles the case where the prnciple is * - aws_principal = [statement.get("Principal")] - else: - aws_principal = statement.get("Principal").get("AWS") - # A principal can either be a single ARN in a string, or an array of ARNs - if isinstance(aws_principal, basestring): - aws_principal = [aws_principal] - - print aws_principal + aws_principal = [aws_principal] + + principal_account_ids = set() for arn in aws_principal: - if arn == "*": - has_issue = True - bad_statements.append(json.dumps(statement)) + if arn == "*" and not condition_accounts: + notes = "An KMS policy where { 'Principal': { 'AWS': '*' } } must also have" + notes += " a {'Condition': {'StringEquals': { 'kms:CallerAccount': '' } } }" + notes += " or it is open to the world." + self.add_issue(5, tag, kms_item, notes=notes) continue if ':' not in arn: @@ -68,9 +99,8 @@ def check_for_kms_policy_with_foreign_account(self, kms_item): statement_account_id = arn.split(":")[4] if statement_account_id != key_account_id: - has_issue = True - bad_statements.append(json.dumps(statement)) + principal_account_ids.add(statement_account_id) - if has_issue: - notes = ", ".join(bad_statements) - self.add_issue(5, tag, kms_item, notes=notes) + if principal_account_ids: + notes = "Principal - {}".format(json.dumps(sorted(list(principal_account_ids)))) + self.add_issue(5, tag, kms_item, notes=notes) diff --git a/security_monkey/auditors/s3.py b/security_monkey/auditors/s3.py index 384b9abe0..9cbe686e0 100644 --- a/security_monkey/auditors/s3.py +++ b/security_monkey/auditors/s3.py @@ -24,8 +24,6 @@ from security_monkey.watchers.s3 import S3 from security_monkey.datastore import Account -import re - class S3Auditor(Auditor): index = S3.index @@ -40,7 +38,7 @@ def check_acl(self, s3_item): S3_ACCOUNT_NAMES = [account.s3_name.lower() for account in accounts if not account.third_party and account.s3_name] S3_THIRD_PARTY_ACCOUNTS = [account.s3_name.lower() for account in accounts if account.third_party and account.s3_name] - acl = s3_item.config.get('grants', {}) + acl = s3_item.config.get('Grants', {}) for user in acl.keys(): if user == 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers': message = "ACL - AuthenticatedUsers USED. " @@ -68,10 +66,11 @@ def check_acl(self, s3_item): self.add_issue(10, message, s3_item, notes=notes) def check_policy(self, s3_item): - policy = s3_item.config.get('policy', {}) + policy = s3_item.config.get('Policy', {}) if not policy: message = "POLICY - No Policy." self.add_issue(0, message, s3_item) + return statements = policy.get('Statement', {}) complained = [] @@ -111,30 +110,28 @@ def inspect_policy_cross_account(self, statement, s3_item, complained): if aws_entry[0:26] not in complained: self.processCrossAccount(aws_entry, s3_item) complained.append(aws_entry[0:26]) - except Exception, e: - print "Exception in cross_account. {} {}".format(Exception, e) + except Exception as e: + print("Exception in cross_account. {} {}".format(Exception, e)) import traceback - print traceback.print_exc() - - def processCrossAccount(self, arn, s3_item): - m = re.match(r'arn:aws:iam::([0-9*]+):', arn) - - # BAD POLICY - Cross Account Access: Bad ARN: * - # "Bad ARN: {}".format(arn) - if not m: - if not '*' == arn: - message = "POLICY - Bad ARN" - notes = "{}".format(arn) - self.add_issue(3, message, s3_item, notes=notes) + print(traceback.print_exc()) + + def processCrossAccount(self, input, s3_item): + from security_monkey.common.arn import ARN + arn = ARN(input) + + if arn.error and input != input: + message = "POLICY - Bad ARN" + notes = "{}".format(arn) + self.add_issue(3, message, s3_item, notes=notes) return # 'WILDCARD ARN: *' # This is caught by check_policy_allow_all(), so ignore here. - if '*' == m.group(1): - print "This is an odd arn: {}".format(arn) + if '*' == arn.account_number: + print("This is an odd arn: {}".format(arn)) return - account = Account.query.filter(Account.number==m.group(1)).first() + account = Account.query.filter(Account.number==arn.account_number).first() if account: # Friendly Account. if not account.third_party: @@ -151,7 +148,7 @@ def processCrossAccount(self, arn, s3_item): # Foreign Unknown Account message = "POLICY - Unknown Cross Account Access." - notes = "Account ID: {} ARN: {}".format(m.group(1), arn) + notes = "Account ID: {} ARN: {}".format(arn.account_number, input) self.add_issue(10, message, s3_item, notes=notes) return diff --git a/security_monkey/auditors/vpc/vpc.py b/security_monkey/auditors/vpc/vpc.py new file mode 100644 index 000000000..add9ae095 --- /dev/null +++ b/security_monkey/auditors/vpc/vpc.py @@ -0,0 +1,55 @@ +# Copyright 2016 Bridgewater Associates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +.. module: security_monkey.auditors.vpc + :platform: Unix + +.. version:: $$VERSION$$ +.. moduleauthor:: Bridgewater OSS + + +""" +from security_monkey.auditor import Auditor +from security_monkey.watchers.vpc.vpc import VPC +from security_monkey.watchers.vpc.flow_log import FlowLog + + +class VPCAuditor(Auditor): + index = VPC.index + i_am_singular = VPC.i_am_singular + i_am_plural = VPC.i_am_plural + support_watcher_indexes = [FlowLog.index] + + def __init__(self, accounts=None, debug=False): + super(VPCAuditor, self).__init__(accounts=accounts, debug=debug) + + def check_flow_logs_enabled(self, vpc_item): + """ + alert when flow logs are not enabled for VPC + """ + flow_log_items = self.get_watcher_support_items( + FlowLog.index, vpc_item.account) + vpc_id = vpc_item.config.get("id") + + tag = "Flow Logs not enabled for VPC" + severity = 5 + + flow_logs_enabled = False + for flow_log in flow_log_items: + if vpc_id == flow_log.config.get("resource_id"): + flow_logs_enabled = True + break + + if not flow_logs_enabled: + self.add_issue(severity, tag, vpc_item) diff --git a/security_monkey/auth/modules.py b/security_monkey/auth/modules.py index d84e16803..ec26b7e66 100644 --- a/security_monkey/auth/modules.py +++ b/security_monkey/auth/modules.py @@ -240,7 +240,7 @@ def _check_permission(self, roles, method, resource): def _deny_hook(self, resource=None): app = self.get_app() - if current_user.is_authenticated(): + if current_user.is_authenticated: status = 403 else: status = 401 @@ -250,7 +250,7 @@ def _deny_hook(self, resource=None): url = "https://{}:{}{}".format(app.config.get('FQDN'), app.config.get('NGINX_PORT'), '/login') else: url = "http://{}:{}{}".format(app.config.get('FQDN'), app.config.get('API_PORT'), '/login') - if current_user.is_authenticated(): + if current_user.is_authenticated: auth_dict = { "authenticated": True, "user": current_user.email, diff --git a/security_monkey/backup.py b/security_monkey/backup.py index a31d22c16..c2517cf35 100644 --- a/security_monkey/backup.py +++ b/security_monkey/backup.py @@ -24,14 +24,16 @@ import json import os + def backup_config_to_json(account_names, monitor_names, output_folder): - monitors = get_monitors(account_names, monitor_names) - for monitor in monitors: - for account in account_names: - _backup_items_in_account(account, monitor, output_folder) + for account_name in account_names: + monitors = get_monitors(account_name, monitor_names) + for monitor in monitors: + _backup_items_in_account(account_name, monitor.watcher, output_folder) + -def _backup_items_in_account(account_name, monitor, output_folder): - technology_name = monitor.watcher.index +def _backup_items_in_account(account_name, watcher, output_folder): + technology_name = watcher.index query = Item.query query = query.join((Account, Account.id==Item.account_id)) query = query.join((Technology, Technology.id==Item.tech_id)) diff --git a/security_monkey/common/PolicyDiff.py b/security_monkey/common/PolicyDiff.py index 0f4ca82d5..f12a50e28 100644 --- a/security_monkey/common/PolicyDiff.py +++ b/security_monkey/common/PolicyDiff.py @@ -35,14 +35,8 @@ def escape(data): return cgi_escape(unicode(data)) -def i(indentlevel): - retstr = '' - for i in range(0, indentlevel): - # The   prevents a user from copying the output from a browser window into anything that requires - # valid JSON. Instead of copying as a tab or space character, it copies as an invalid whitespace character HEX E28083 - #retstr += ' ' - retstr += '    ' - return retstr +def i(indentation): + return '    '*indentation # ADDED @@ -50,67 +44,51 @@ def i(indentlevel): # Type Change # Regular Change # DELETED -def process_sub_dict(key, sda, sdb, indentlevel): +def process_sub_dict(key, sda, sdb, indentation): if type(sda) is not type(sdb): raise ValueError("process_sub_dict requires that both items have the same type.") # BUG: What if going from None to 'vpc-1de23c' retstr = '' - brackets = charfortype(sda) - if type(sda) is str or type(sdb) is unicode: + brackets = get_brackets(sda) + if type(sda) in [str, unicode]: if sda == sdb: - retstr += same("{4}\"{0}\": {2}{1}{3},".format(key, sda, brackets[0], brackets[1], i(indentlevel))) + retstr += same("{4}\"{0}\": {2}{1}{3},".format(key, escape(sda), brackets['open'], brackets['close'], i(indentation))) else: - retstr += deleted("{4}\"{0}\": {2}{1}{3},".format(key, sdb, brackets[0], brackets[1], i(indentlevel))) - retstr += added("{4}\"{0}\": {2}{1}{3},".format(key, sda, brackets[0], brackets[1], i(indentlevel))) - elif type(sda) is type(None) or type(sda) is bool or type(sda) is int or type(sda) is float: + retstr += deleted("{4}\"{0}\": {2}{1}{3},".format(key, escape(sdb), brackets['open'], brackets['close'], i(indentation))) + retstr += added("{4}\"{0}\": {2}{1}{3},".format(key, escape(sda), brackets['open'], brackets['close'], i(indentation))) + elif type(sda) in [bool, type(None), int, float]: if sda == sdb: - retstr += same("{2}\"{0}\": {1},".format(key, json.dumps(sda), i(indentlevel))) + retstr += same("{2}\"{0}\": {1},".format(key, json.dumps(sda), i(indentation))) else: - retstr += deleted("{2}\"{0}\": {1},".format(key, json.dumps(sda), i(indentlevel))) - retstr += added("{2}\"{0}\": {1},".format(key, json.dumps(sdb), i(indentlevel))) + retstr += deleted("{2}\"{0}\": {1},".format(key, json.dumps(sdb), i(indentation))) + retstr += added("{2}\"{0}\": {1},".format(key, json.dumps(sda), i(indentation))) elif type(sda) is dict: - retstr += same("{4}\"{0}\": {2}
    \n{1}{4}{3},".format(key, diffdict(sda, sdb, indentlevel+1), brackets[0], brackets[1], i(indentlevel))) + retstr += same("{4}\"{0}\": {2}
    \n{1}{4}{3},".format(key, diff_dict(sda, sdb, indentation + 1), brackets['open'], brackets['close'], i(indentation))) elif type(sda) is list: - retstr += same("{4}\"{0}\": {2}
    \n{1}{4}{3},".format(key, difflist(sda, sdb, indentlevel+1), brackets[0], brackets[1], i(indentlevel))) + retstr += same("{4}\"{0}\": {2}
    \n{1}{4}{3},".format(key, diff_list(sda, sdb, indentation + 1), brackets['open'], brackets['close'], i(indentation))) else: - print "process_sub_dict - Unexpected diffdict type {}".format(type(sda)) + print "process_sub_dict - Unexpected type {}".format(type(sda)) return retstr -def formbrack(value, indentlevel): - brackets = {'open': '', 'close': ''} - - if type(value) is str or type(value) is unicode: - brackets['open'] = '"' - brackets['close'] = '"' - elif type(value) is dict: - brackets['open'] = '{
    \n' - brackets['close'] = i(indentlevel)+'}' - elif type(value) is list: - brackets['open'] = '[
    \n' - brackets['close'] = i(indentlevel)+']' - else: - # print "formbrack - Unexpected diffdict type {}".format(type(value)) - pass - return brackets - - -def printlist(structure, action, indentlevel): +def print_list(structure, action, indentation): retstr = '' for value in structure: - brackets = formbrack(value, indentlevel) + brackets = form_brackets(value, indentation) new_value = "" - if type(value) is str or type(value) is unicode: + if type(value) in [str, unicode, int, float]: new_value = escape(value) + elif type(value) in [bool, type(None)]: + new_value = json.dumps(value) elif type(value) is dict: - new_value = printdict(value, action, indentlevel+1) + new_value = print_dict(value, action, indentation + 1) elif type(value) is list: - new_value = printlist(value, action, indentlevel+1) + new_value = print_list(value, action, indentation + 1) else: - print "printlist - Unexpected diffdict type {}".format(type(value)) + print "print_list - Unexpected type {}".format(type(value)) - content = "{3}{1}{0}{2},".format(new_value, brackets['open'], brackets['close'], i(indentlevel)) + content = "{3}{1}{0}{2},".format(new_value, brackets['open'], brackets['close'], i(indentation)) if action is 'same': retstr += same(content) @@ -121,29 +99,29 @@ def printlist(structure, action, indentlevel): return remove_last_comma(retstr) -def printdict(structure, action, indentlevel): +def print_dict(structure, action, indentation): retstr = '' for key in structure.keys(): value = structure[key] - brackets = formbrack(value, indentlevel) + brackets = form_brackets(value, indentation) new_value = '' - if type(value) is str or type(value) is unicode or type(value) is int or type(value) is float: + if type(value) in [str, unicode, int, float]: new_value = escape(value) - elif type(value) is bool or type(value) is type(None): + elif type(value) in [bool, type(None)]: new_value = json.dumps(value) elif type(value) is dict: - new_value = printdict(value, action, indentlevel+1) + new_value = print_dict(value, action, indentation + 1) elif type(value) is list: - new_value = printlist(value, action, indentlevel+1) + new_value = print_list(value, action, indentation + 1) else: - print "printdict - Unexpected diffdict type {}".format(type(value)) + print "print_dict - Unexpected type {}".format(type(value)) content = "{4}\"{0}\": {2}{1}{3},".format( escape(key), new_value, brackets['open'], brackets['close'], - i(indentlevel) + i(indentation) ) if action is 'same': @@ -155,48 +133,47 @@ def printdict(structure, action, indentlevel): return remove_last_comma(retstr) -def printsomething(value, action, indentlevel): - if type(value) is str or type(value) is unicode or type(value) is int or type(value) is float: +def print_item(value, action, indentlevel): + if type(value) in [str, unicode, int, float]: return escape(value) - elif type(value) is bool or type(value) is type(None): - new_value = json.dumps(value) + elif type(value) in [bool, type(None)]: + return json.dumps(value) elif type(value) is dict: - return printdict(value, action, indentlevel) + return print_dict(value, action, indentlevel) elif type(value) is list: - return printlist(value, action, indentlevel) + return print_list(value, action, indentlevel) else: - print "printsomething - Unexpected diffdict type {}".format(type(value)) + print "print_item - Unexpected diff_dict type {}".format(type(value)) return '' -def diffdict(dicta, dictb, indentlevel): +def diff_dict(dicta, dictb, indentation): """ - diffdict and difflist are recursive methods which build an HTML representation of the differences between two objects. - TODO: diffdict does not add commas + diff_dict and diff_list are recursive methods which build an HTML representation of the differences between two objects. """ retstr = '' for keya in dicta.keys(): if not keya in dictb: - brackets = charfortype(dicta[keya]) - if type(dicta[keya]) is str or type(dicta[keya]) is unicode: - retstr += added("{4}\"{0}\": {2}{1}{3},".format(keya, printsomething(dicta[keya], 'added', indentlevel+1), brackets[0], brackets[1], i(indentlevel))) - if type(dicta[keya]) is list or type(dicta[keya]) is dict: - retstr += added("{4}\"{0}\": {2}
    \n{1}{4}{3},".format(keya, printsomething(dicta[keya], 'added', indentlevel+1), brackets[0], brackets[1], i(indentlevel))) + brackets = get_brackets(dicta[keya]) + if type(dicta[keya]) in [str, unicode, int, float, bool, type(None)]: + retstr += added("{4}\"{0}\": {2}{1}{3},".format(keya, print_item(dicta[keya], 'added', indentation + 1), brackets['open'], brackets['close'], i(indentation))) + if type(dicta[keya]) in [list, dict]: + retstr += added("{4}\"{0}\": {2}
    \n{1}{4}{3},".format(keya, print_item(dicta[keya], 'added', indentation + 1), brackets['open'], brackets['close'], i(indentation))) else: if not type(dicta[keya]) is type(dictb[keya]): - brackets = charfortype(dictb[keya]) - retstr += deleted("{4}\"{0}\": {2}{1}{3},".format(keya, dictb[keya], brackets[0], brackets[1], i(indentlevel))) - brackets = charfortype(dicta[keya]) - retstr += added("{4}\"{0}\": {2}{1}{3},".format(keya, dicta[keya], brackets[0], brackets[1], i(indentlevel))) + brackets = get_brackets(dictb[keya]) + retstr += deleted("{4}\"{0}\": {2}{1}{3},".format(keya, dictb[keya], brackets['open'], brackets['close'], i(indentation))) + brackets = get_brackets(dicta[keya]) + retstr += added("{4}\"{0}\": {2}{1}{3},".format(keya, dicta[keya], brackets['open'], brackets['close'], i(indentation))) else: - retstr += process_sub_dict(keya, dicta[keya], dictb[keya], indentlevel) + retstr += process_sub_dict(keya, dicta[keya], dictb[keya], indentation) for keyb in dictb.keys(): if not keyb in dicta: - brackets = charfortype(dictb[keyb]) - if type(dictb[keyb]) is str or type(dictb[keyb]) is unicode: - retstr += deleted("{4}\"{0}\": {2}{1}{3},".format(keyb, printsomething(dictb[keyb], 'deleted', indentlevel+1), brackets[0], brackets[1], i(indentlevel))) - if type(dictb[keyb]) is list or type(dictb[keyb]) is dict: - retstr += deleted("{4}\"{0}\": {2}
    \n{1}{4}{3},".format(keyb, printsomething(dictb[keyb], 'deleted', indentlevel+1), brackets[0], brackets[1], i(indentlevel))) + brackets = get_brackets(dictb[keyb]) + if type(dictb[keyb]) in [str, unicode, int, float, bool, type(None)]: + retstr += deleted("{4}\"{0}\": {2}{1}{3},".format(keyb, print_item(dictb[keyb], 'deleted', indentation + 1), brackets['open'], brackets['close'], i(indentation))) + if type(dictb[keyb]) in [list, dict]: + retstr += deleted("{4}\"{0}\": {2}
    \n{1}{4}{3},".format(keyb, print_item(dictb[keyb], 'deleted', indentation + 1), brackets['open'], brackets['close'], i(indentation))) return remove_last_comma(retstr) @@ -205,10 +182,9 @@ def remove_last_comma(str): return str[:position] + str[position+1:] -def difflist(lista, listb, indentlevel): +def diff_list(lista, listb, indentation): """ - diffdict and difflist are recursive methods which build an HTML representation of the differences between two objects. - TODO: difflist adds commas after every entry, even the last entry. + diff_dict and diff_list are recursive methods which build an HTML representation of the differences between two objects. When multiple items in a list have been modified, the levenshtein distance is used to find which items most likely were modified from one object to the next. Although this may not be necessary when the list items are primatives @@ -232,60 +208,69 @@ def difflist(lista, listb, indentlevel): for item in lista: if item in listb: - brackets = charfortype(item) - if type(item) is str or type(item) is unicode: - retstr += same("{3}{1}{0}{2},".format(escape(item), brackets[0], brackets[1], i(indentlevel))) + brackets = get_brackets(item) + if type(item) in [str, unicode, int, float]: + retstr += same("{3}{1}{0}{2},".format(escape(item), brackets['open'], brackets['close'], i(indentation))) + elif type(item) in [bool, type(None)]: + retstr += same("{3}{1}{0}{2},".format(json.dumps(item), brackets['open'], brackets['close'], i(indentation))) + elif type(item) in [list, dict]: + diffstr = print_item(item, 'same', indentation + 1) + retstr += same("{3}{1}
    \n{0}{3}{2},".format(diffstr, brackets['open'], brackets['close'], i(indentation))) else: - # Handle lists and dicts here: - diffstr = '' - if type(item) is list or type(item) is dict: - diffstr = printsomething(item, 'same', indentlevel+1) - retstr += same("{3}{1}
    \n{0}{3}{2},".format(diffstr, brackets[0], brackets[1], i(indentlevel))) + print "diff_list - Unexpected Type {}".format(type(item)) else: addedlist.append(item) + for item in listb: - if not item in lista: + if item not in lista: deletedlist.append(item) + for item in addedlist: - bestmatch = findmostsimilar(item, deletedlist) - brackets = charfortype(item) + bestmatch = find_most_similar(item, deletedlist) + brackets = get_brackets(item) if None is bestmatch: - if type(item) is str or type(item) is unicode: - retstr += added("{3}{1}{0}{2},".format(escape(item), brackets[0], brackets[1], i(indentlevel))) + if type(item) in [str, unicode, int, float]: + retstr += added("{3}{1}{0}{2},".format(escape(item), brackets['open'], brackets['close'], i(indentation))) + elif type(item) in [bool, type(None)]: + retstr += added("{3}{1}{0}{2},".format(json.dumps(item), brackets['open'], brackets['close'], i(indentation))) + elif type(item) in [list, dict]: + diffstr = print_item(item, 'added', indentation + 1) + retstr += added("{3}{1}
    \n{0}{3}{2},".format(diffstr, brackets['open'], brackets['close'], i(indentation))) else: - # Handle lists and dicts here: - diffstr = '' - if type(item) is list or type(item) is dict: - diffstr = printsomething(item, 'added', indentlevel+1) - retstr += added("{3}{1}
    \n{0}{3}{2},".format(diffstr, brackets[0], brackets[1], i(indentlevel))) + print "diff_list - Unexpected Type {}".format(type(item)) else: - if type(item) is str or type(item) is unicode: - retstr += deleted("{3}{1}{0}{2},".format(escape(bestmatch), brackets[0], brackets[1], i(indentlevel))) - retstr += added("{3}{1}{0}{2},".format(escape(item), brackets[0], brackets[1], i(indentlevel))) + if type(item) in [str, unicode, int, float]: + retstr += deleted("{3}{1}{0}{2},".format(escape(bestmatch), brackets['open'], brackets['close'], i(indentation))) + retstr += added("{3}{1}{0}{2},".format(escape(item), brackets['open'], brackets['close'], i(indentation))) + elif type(item) in [bool, type(None)]: + retstr += deleted("{3}{1}{0}{2},".format(json.dumps(bestmatch), brackets['open'], brackets['close'], i(indentation))) + retstr += added("{3}{1}{0}{2},".format(json.dumps(item), brackets['open'], brackets['close'], i(indentation))) + elif type(item) is list: + diffstr = diff_list(item, bestmatch, indentation + 1) + retstr += same("{3}{1}
    \n{0}{3}{2},".format(diffstr, brackets['open'], brackets['close'], i(indentation))) + elif type(item) is dict: + diffstr = diff_dict(item, bestmatch, indentation + 1) + retstr += same("{3}{1}
    \n{0}{3}{2},".format(diffstr, brackets['open'], brackets['close'], i(indentation))) else: - # Handle lists and dicts here: - diffstr = '' - if type(item) is list: - diffstr = difflist(item, bestmatch, indentlevel+1) - elif type(item) is dict: - diffstr = diffdict(item, bestmatch, indentlevel+1) - retstr += same("{3}{1}
    \n{0}{3}{2},".format(diffstr, brackets[0], brackets[1], i(indentlevel))) + print "diff_list - Unexpected Type {}".format(type(item)) deletedlist.remove(bestmatch) + for item in deletedlist: - brackets = charfortype(item) - if type(item) is str or type(item) is unicode: - retstr += deleted("{3}{1}{0}{2},".format(escape(item), brackets[0], brackets[1], i(indentlevel))) + brackets = get_brackets(item) + if type(item) in [str, unicode, int, float]: + retstr += deleted("{3}{1}{0}{2},".format(escape(item), brackets['open'], brackets['close'], i(indentation))) + elif type(item) in [bool, type(None)]: + retstr += deleted("{3}{1}{0}{2},".format(json.dumps(item), brackets['open'], brackets['close'], i(indentation))) + elif type(item) in [list, dict]: + diffstr = print_item(item, 'deleted', indentation + 1) + retstr += deleted("{3}{1}
    \n{0}{3}{2},".format(diffstr, brackets['open'], brackets['close'], i(indentation))) else: - # Handle lists and dicts here: - diffstr = '' - if type(item) is list or type(item) is dict: - diffstr = printsomething(item, 'deleted', indentlevel+1) - retstr += deleted("{3}{1}
    \n{0}{3}{2},".format(diffstr, brackets[0], brackets[1], i(indentlevel))) + print "diff_list - Unexpected Type {}".format(type(item)) return remove_last_comma(retstr) # levenshtein - http://hetland.org/coding/python/levenshtein.py -def strdistance(a, b): +def str_distance(a, b): "Calculates the Levenshtein distance between a and b." n, m = len(a), len(b) if n > m: @@ -304,7 +289,7 @@ def strdistance(a, b): return current[n] -def findmostsimilar(item, list): +def find_most_similar(item, list): stritem = str(item) mindistance = sys.maxint bestmatch = None @@ -317,7 +302,7 @@ def findmostsimilar(item, list): for listitem in list: if type(listitem) == type(item): strlistitem = str(listitem) - distance = strdistance(stritem, strlistitem) + distance = str_distance(stritem, strlistitem) if distance == 0: return listitem if distance < mindistance: @@ -326,14 +311,34 @@ def findmostsimilar(item, list): return bestmatch -def charfortype(obj): - if type(obj) is str or type(obj) is unicode: - return "\"\"" - if type(obj) is list: - return "[]" - if type(obj) is dict: - return "{}" - return " " +def form_brackets(value, indentation): + brackets = {'open': '', 'close': ''} + + if type(value) in [str, unicode]: + brackets['open'] = '"' + brackets['close'] = '"' + elif type(value) is dict: + brackets['open'] = '{
    \n' + brackets['close'] = i(indentation) + '}' + elif type(value) is list: + brackets['open'] = '[
    \n' + brackets['close'] = i(indentation) + ']' + return brackets + + +def get_brackets(item): + brackets = {'open': '', 'close': ''} + + if type(item) in [str, unicode]: + brackets['open'] = '"' + brackets['close'] = '"' + if type(item) is list: + brackets['open'] = '[' + brackets['close'] = ']' + if type(item) is dict: + brackets['open'] = '{' + brackets['close'] = '}' + return brackets def color(text, color): @@ -353,6 +358,17 @@ def same(text): class PolicyDiff(object): + """ + old_pol = "{}" + new_pol = { + "create_date": "2013-09-12T18:28:21Z", + "must_change_password": "false", + "user_name": "test" + } + + differ = PolicyDiff(new_pol, old_pol) + print differ.produceDiffHTML() + """ def __init__(self, new_policy, old_policy): self._new_policy = None @@ -364,6 +380,8 @@ def __init__(self, new_policy, old_policy): except: print "Could not read policy in as json. Type: {} Policy: {}".format(type(new_policy), new_policy) self._new_policy = new_policy + else: + self._new_policy = json.loads(json.dumps(new_policy)) if isinstance(old_policy, basestring): try: @@ -371,16 +389,8 @@ def __init__(self, new_policy, old_policy): except: print "Could not read policy in as json. Type: {} Policy: {}".format(type(old_policy), old_policy) self._old_policy = old_policy - - if type(new_policy) is list or isinstance(new_policy, dict): - self._new_policy = new_policy - if type(self._new_policy) is collections.defaultdict: - self._new_policy = dict(self._new_policy) - - if type(old_policy) is list or isinstance(old_policy, dict): - self._old_policy = old_policy - if type(self._old_policy) is collections.defaultdict: - self._old_policy = dict(self._old_policy) + else: + self._old_policy = json.loads(json.dumps(old_policy)) if self._old_policy is None or self._new_policy is None: raise ValueError("PolicyDiff could not process old policy or new policy or both.") @@ -404,28 +414,13 @@ def produceDiffHTML(self): if isinstance(self._old_policy, basestring): return "{0}
    {1}".format(deleted(self._old_policy), added(self._new_policy)) - brackets = charfortype(self._new_policy) - inner_html = None + brackets = get_brackets(self._new_policy) if type(self._new_policy) is dict: - inner_html = diffdict(self._new_policy, self._old_policy, 1) + inner_html = diff_dict(self._new_policy, self._old_policy, 1) elif type(self._new_policy) is list: - inner_html = difflist(self._new_policy, self._old_policy, 1) + inner_html = diff_list(self._new_policy, self._old_policy, 1) else: raise ValueError("PolicyDiff::produceDiffHTML cannot process items of type: {}".format(type(self._new_policy))) - return "{1}
    \n{0}{2}
    \n".format(inner_html, brackets[0], brackets[1]) - -if __name__ == "__main__": - - old_pol = "{}" - new_pol = """ - { - "create_date": "2013-09-12T18:28:21Z", - "must_change_password": "false", - "user_name": "test" - } - """ - - pdiddy = PolicyDiff(new_pol, old_pol) - print pdiddy.produceDiffHTML() + return "{1}
    \n{0}{2}
    \n".format(inner_html, brackets['open'], brackets['close']) diff --git a/security_monkey/common/arn.py b/security_monkey/common/arn.py index 71942ff44..9830f1ff2 100644 --- a/security_monkey/common/arn.py +++ b/security_monkey/common/arn.py @@ -35,7 +35,7 @@ class ARN(object): service = False def __init__(self, input): - arn_match = re.search('^arn:([^:]*):([^:]*):([^:]*):(|\*|[\d]{12}):(.+)$', input) + arn_match = re.search('^arn:([^:]*):([^:]*):([^:]*):(|\*|[\d]{12}|cloudfront):(.+)$', input) if arn_match: if arn_match.group(2) == "iam" and arn_match.group(5) == "root": self.root = True diff --git a/security_monkey/common/utils.py b/security_monkey/common/utils.py index 29174b0f1..dcff480d2 100644 --- a/security_monkey/common/utils.py +++ b/security_monkey/common/utils.py @@ -112,7 +112,9 @@ def add_account(number, third_party, name, s3_name, active, notes, role_name='Se if not edit: return False else: - query.delete() + account = query.first() + db.session.delete(account) + db.session.commit() account = Account() account.name = name account.s3_name = s3_name diff --git a/security_monkey/datastore.py b/security_monkey/datastore.py index 31bf8bf58..5f393b19f 100644 --- a/security_monkey/datastore.py +++ b/security_monkey/datastore.py @@ -32,7 +32,9 @@ from sqlalchemy import Column, Integer, String, DateTime, Boolean, Unicode, Text from sqlalchemy.dialects.postgresql import CIDR from sqlalchemy.schema import ForeignKey, UniqueConstraint -from sqlalchemy.orm import relationship, backref +from sqlalchemy.orm import relationship, backref, column_property +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy import select, func from sqlalchemy.orm import deferred @@ -54,6 +56,16 @@ ) +class AccountType(db.Model): + """ + Defines the type of account based on where the data lives, e.g. AWS. + """ + __tablename__ = "account_type" + id = Column(Integer, primary_key=True) + name = db.Column(db.String(80), unique=True) + accounts = relationship("Account", backref="account_type") + + class Account(db.Model): """ Meant to model AWS accounts. @@ -62,16 +74,38 @@ class Account(db.Model): id = Column(Integer, primary_key=True) active = Column(Boolean()) third_party = Column(Boolean()) - name = Column(String(32)) + name = Column(String(32), unique=True) + s3_name = Column(String(64)) # (deprecated-custom) + number = Column(String(12)) # (deprecated-identifier) Not stored as INT because of potential leading-zeros. notes = Column(String(256)) - s3_name = Column(String(64)) - number = Column(String(12)) # Not stored as INT because of potential leading-zeros. + identifier = Column(String(256)) # Unique id of the account, the number for AWS. items = relationship("Item", backref="account", cascade="all, delete, delete-orphan") issue_categories = relationship("AuditorSettings", backref="account") - role_name = Column(String(256)) + role_name = Column(String(256)) # (deprecated-custom) + account_type_id = Column(Integer, ForeignKey("account_type.id"), nullable=False) + custom_fields = relationship("AccountTypeCustomValues", lazy="immediate", cascade="all, delete, delete-orphan") + unique_const = UniqueConstraint('account_type_id', 'identifier') exceptions = relationship("ExceptionLogs", backref="account", cascade="all, delete, delete-orphan") + def getCustom(self, name): + for field in self.custom_fields: + if field.name == name: + return field.value + return None + + +class AccountTypeCustomValues(db.Model): + """ + Defines the values for custom fields defined in AccountTypeCustomFields. + """ + __tablename__ = "account_type_values" + id = Column(Integer, primary_key=True) + name = Column(db.String(64)) + value = db.Column(db.String(256)) + account_id = Column(Integer, ForeignKey("account.id"), nullable=False) + unique_const = UniqueConstraint('account_id', 'name') + class Technology(db.Model): """ @@ -136,6 +170,10 @@ class User(UserMixin, db.Model, RBACUserMixin): def __str__(self): return '' % (self.id, self.email) +issue_item_association = db.Table('issue_item_association', + Column('super_issue_id', Integer, ForeignKey('itemaudit.id')), + Column('sub_item_id', Integer, ForeignKey('item.id')) +) class ItemAudit(db.Model): """ @@ -145,13 +183,14 @@ class ItemAudit(db.Model): id = Column(Integer, primary_key=True) score = Column(Integer) issue = Column(String(512)) - notes = Column(String(512)) + notes = Column(String(1024)) justified = Column(Boolean) justified_user_id = Column(Integer, ForeignKey("user.id"), nullable=True, index=True) justification = Column(String(512)) justified_date = Column(DateTime(), default=datetime.datetime.utcnow, nullable=True) item_id = Column(Integer, ForeignKey("item.id"), nullable=False, index=True) auditor_setting_id = Column(Integer, ForeignKey("auditorsettings.id"), nullable=True, index=True) + sub_items = relationship("Item", secondary=issue_item_association, backref="super_issues") def __str__(self): return "Issue: [{issue}] Score: {score} Justified: {justified}\nNotes: {notes}\n".format( @@ -186,8 +225,7 @@ class Item(db.Model): """ __tablename__ = "item" id = Column(Integer, primary_key=True) - cloud = Column(String(32)) # AWS, Google, Other - region = Column(String(32)) + region = Column(String(32), index=True) name = Column(String(303), index=True) # Max AWS name = 255 chars. Add 48 chars for ' (sg-12345678901234567 in vpc-12345678901234567)' arn = Column(Text(), nullable=True, index=True, unique=True) latest_revision_complete_hash = Column(String(32), index=True) @@ -199,8 +237,55 @@ class Item(db.Model): revisions = relationship("ItemRevision", backref="item", cascade="all, delete, delete-orphan", order_by="desc(ItemRevision.date_created)", lazy="dynamic") issues = relationship("ItemAudit", backref="item", cascade="all, delete, delete-orphan") cloudtrail_entries = relationship("CloudTrailEntry", backref="item", cascade="all, delete, delete-orphan", order_by="CloudTrailEntry.event_time") - + issues = relationship("ItemAudit", backref="item", cascade="all, delete, delete-orphan", foreign_keys="ItemAudit.item_id") exceptions = relationship("ExceptionLogs", backref="item", cascade="all, delete, delete-orphan") + + @hybrid_property + def score(self): + return db.session.query( + func.cast( + func.sum(ItemAudit.score), + Integer) + ).filter( + ItemAudit.item_id == self.id, + ItemAudit.auditor_setting_id == AuditorSettings.id, + AuditorSettings.disabled == False).one()[0] or 0 + + @score.expression + def score(cls): + return select([func.sum(ItemAudit.score)]). \ + where(ItemAudit.item_id == cls.id). \ + where(ItemAudit.auditor_setting_id == AuditorSettings.id). \ + where(AuditorSettings.disabled == False). \ + label('item_score') + + @hybrid_property + def unjustified_score(self): + return db.session.query( + func.cast( + func.sum(ItemAudit.score), + Integer) + ).filter( + ItemAudit.item_id == self.id, + ItemAudit.justified == False, + ItemAudit.auditor_setting_id == AuditorSettings.id, + AuditorSettings.disabled == False).one()[0] or 0 + + @unjustified_score.expression + def unjustified_score(cls): + return select([func.sum(ItemAudit.score)]). \ + where(ItemAudit.item_id == cls.id). \ + where(ItemAudit.justified == False). \ + where(ItemAudit.auditor_setting_id == AuditorSettings.id). \ + where(AuditorSettings.disabled == False). \ + label('item_unjustified_score') + + issue_count = column_property( + select([func.count(ItemAudit.id)]) + .where(ItemAudit.item_id == id) + .where(ItemAudit.auditor_setting_id == AuditorSettings.id) + .where(AuditorSettings.disabled == False) + ) class ItemComment(db.Model): @@ -567,14 +652,15 @@ def store_exception(source, location, exception, ttl=None): if account: exception_entry.account_id = account.id - technology = Technology.query.filter(Technology.name == location[0]).first() - if not technology: - technology = Technology(name=location[0]) - db.session.add(technology) - db.session.commit() - db.session.refresh(technology) - - if technology: + if len(location) >= 1: + technology = Technology.query.filter(Technology.name == location[0]).first() + if not technology: + technology = Technology(name=location[0]) + db.session.add(technology) + db.session.commit() + db.session.refresh(technology) + app.logger.info("Creating a new Technology: {} - ID: {}" + .format(technology.name, technology.id)) exception_entry.tech_id = technology.id db.session.add(exception_entry) diff --git a/security_monkey/decorators.py b/security_monkey/decorators.py index a096a9fd1..0b4f47a35 100644 --- a/security_monkey/decorators.py +++ b/security_monkey/decorators.py @@ -76,19 +76,28 @@ def wrapped_function(*args, **kwargs): return decorator -def record_exception(source="boto"): +def record_exception(source="boto", pop_exception_fields=False): def decorator(f): @wraps(f) def decorated_function(*args, **kwargs): + # prevent these from being passed to the wrapped function: + m = kwargs.pop if pop_exception_fields else kwargs.get + exception_values = { + 'index': m('index'), + 'account': m('account_name', None), + 'exception_record_region': m('exception_record_region', None), + 'name': m('name', None), + 'exception_map': m('exception_map') + } try: return f(*args, **kwargs) except Exception as e: - index = kwargs.get('index') - account = kwargs.get('account_name') + index = exception_values['index'] + account = exception_values['account'] # Allow the recording region to be overridden for universal tech like IAM - region = kwargs.get('exception_record_region') or kwargs.get('region') - name = kwargs.get('name') - exception_map = kwargs.get('exception_map') + region = exception_values['exception_record_region'] or kwargs.get('region') + name = exception_values['name'] + exception_map = exception_values['exception_map'] exc = BotoConnectionIssue(str(e), index, account, name) if name: location = (index, account, region, name) @@ -149,7 +158,7 @@ def decorated_function(*args, **kwargs): def get_regions(account, service_name): if not service_name: - return None, ['aws-global'] + return None, ['us-east-1'] sts = boto3.client('sts') role_name = 'SecurityMonkey' diff --git a/security_monkey/jirasync.py b/security_monkey/jirasync.py index 59a509f09..f07eb5174 100644 --- a/security_monkey/jirasync.py +++ b/security_monkey/jirasync.py @@ -75,7 +75,7 @@ def add_or_update_issue(self, issue, technology, account, count): jql = 'project={0} and summary~"{1}"'.format(self.project, summary_search) issues = self.client.search_issues(jql) - url = "{0}/#/issues/-/{1}/{2}/-/True/{3}/1/25".format(self.url, technology, account, urllib.quote(issue, '')) + url = "{0}/#/issues/-/{1}/{2}/-/-/True/{3}/1/25".format(self.url, technology, account, urllib.quote(issue, '')) timezone = time.tzname[time.localtime().tm_isdst] description = ("This ticket was automatically created by Security Monkey. DO NOT EDIT SUMMARY OR BELOW THIS LINE\n" "Number of issues: {0}\n" diff --git a/security_monkey/monitors.py b/security_monkey/monitors.py index 57b4bae7a..5f049c922 100644 --- a/security_monkey/monitors.py +++ b/security_monkey/monitors.py @@ -10,37 +10,115 @@ from security_monkey import app from security_monkey.auditor import auditor_registry from security_monkey.watcher import watcher_registry +from security_monkey.account_manager import account_registry, get_account_by_name class Monitor(object): """Collects a watcher with the associated auditors""" - def __init__(self, watcher_class, accounts, debug=False): - self.watcher = watcher_class(accounts=accounts, debug=debug) + def __init__(self, watcher_class, account, debug=False): + self.watcher = watcher_class(accounts=[account.name], debug=debug) self.auditors = [] + self.audit_tier = 0 + for auditor_class in auditor_registry[self.watcher.index]: - self.auditors.append(auditor_class(accounts=accounts, debug=debug)) + au = auditor_class([account.name], debug=debug) + if au.applies_to_account(account): + self.auditors.append(au) + -def get_monitors(accounts, monitor_names, debug=False): +def get_monitors(account_name, monitor_names, debug=False): """ Returns a list of monitors in the correct audit order which apply to one or more of the accounts. """ requested_mons = [] + account = get_account_by_name(account_name) + account_manager = account_registry.get(account.account_type.name)() + for monitor_name in monitor_names: watcher_class = watcher_registry[monitor_name] - monitor = Monitor(watcher_class, accounts, debug) - requested_mons.append(monitor) + if account_manager.is_compatible_with_account_type(watcher_class.account_type): + monitor = Monitor(watcher_class, account, debug) + requested_mons.append(monitor) + + return requested_mons + + +def get_monitors_and_dependencies(account, monitor_names, debug=False): + """ + Returns a list of monitors in the correct audit order which apply to one or + more of the accounts plus any monitors with audit results dependent on the + ones requested. + """ + monitors = all_monitors(account, debug) + monitor_names = _find_dependent_monitors(monitors, monitor_names) + requested_mons = [] + for mon in monitors: + if mon.watcher.index in monitor_names: + requested_mons.append(mon) return requested_mons -def all_monitors(accounts, debug=False): + +def all_monitors(account_name, debug=False): """ Returns a list of all monitors in the correct audit order which apply to one or more of the accounts. """ - monitors = [] + monitor_dict = {} + account = get_account_by_name(account_name) + account_manager = account_registry.get(account.account_type.name)() for watcher_class in watcher_registry.itervalues(): - monitor = Monitor(watcher_class, accounts, debug) - monitors.append(monitor) + if account_manager.is_compatible_with_account_type(watcher_class.account_type): + monitor = Monitor(watcher_class, account, debug) + monitor_dict[monitor.watcher.index] = monitor + for mon in monitor_dict.values(): + if len(mon.auditors) > 0: + path = [ mon.watcher.index ] + _set_dependency_hierarchies(monitor_dict, mon, path, mon.audit_tier + 1) + + monitors = sorted(monitor_dict.values(), key=lambda item: item.audit_tier, reverse=True) return monitors + + +def _set_dependency_hierarchies(monitor_dict, monitor, path, level): + declared_support_indexes = set() + + for auditor in monitor.auditors: + declared_support_indexes |= set(auditor.support_auditor_indexes) + + for support_index in declared_support_indexes: + current_path = path + [ support_index ] + if support_index in path: + auditor_flow = '' + for index in current_path: + auditor_flow = auditor_flow + '->' + index + raise Exception('Detected circular dependency in support auditor', auditor_flow) + + support_mon = monitor_dict[support_index] + if support_mon.audit_tier < level: + support_mon.audit_tier = level + + _set_dependency_hierarchies(monitor_dict, support_mon, current_path, level + 1) + +def _find_dependent_monitors(monitors, monitor_names): + """ + Used to find all the monitors that re dependent on those in the original + monitor_names to the list. + """ + last_iteration_count = 0 + while len(monitor_names) != last_iteration_count: + # May need to loop through the list mutiple times in the case of + # chaining dependencies + last_iteration_count = len(monitor_names) + for mon in monitors: + for auditor in mon.auditors: + for support_index in auditor.support_auditor_indexes: + if support_index in monitor_names and mon.watcher.index not in monitor_names: + monitor_names.append(mon.watcher.index) + for support_index in auditor.support_watcher_indexes: + if support_index in monitor_names and mon.watcher.index not in monitor_names: + monitor_names.append(mon.watcher.index) + + return monitor_names diff --git a/security_monkey/reporter.py b/security_monkey/reporter.py index ff8c9be35..8d6a16f6d 100644 --- a/security_monkey/reporter.py +++ b/security_monkey/reporter.py @@ -39,7 +39,7 @@ def __init__(self, account=None, alert_accounts=None, debug=False): alert_accounts = [account] self.account_watchers[account] = [] - for monitor in all_monitors([account]): + for monitor in all_monitors(account, debug): self.account_watchers[account].append((monitor)) if account in alert_accounts: @@ -49,18 +49,23 @@ def run(self, account, interval=None): """Starts the process of watchers -> auditors -> alerters -> watchers.save()""" app.logger.info("Starting work on account {}.".format(account)) time1 = time.time() - for monitor in self.get_watchauditors(account, interval): - app.logger.info("Running {} for {} ({} minutes interval)".format(monitor.watcher.i_am_singular, account, interval)) - (items, exception_map) = monitor.watcher.slurp() - monitor.watcher.find_changes(current=items, exception_map=exception_map) - items_to_audit = [item for item in monitor.watcher.created_items + monitor.watcher.changed_items] + mons = self.get_watchauditors(account, interval) + unique_watcher_results = self.slurp_unique_watchers(mons, account, interval) + + monitors_with_changes = set() + for monitor in mons: + slurped_watcher_results = unique_watcher_results.get(monitor.watcher.index) + monitor.watcher.find_changes(slurped_watcher_results[0], slurped_watcher_results[1]) + if (len(monitor.watcher.created_items) > 0) or (len(monitor.watcher.changed_items) > 0): + monitors_with_changes.add(monitor.watcher.index) + monitor.watcher.save() - if len(items_to_audit) > 0: - for auditor in monitor.auditors: - auditor.audit_these_objects(items_to_audit) - auditor.save_issues() + for monitor in mons: + for auditor in monitor.auditors: + items_to_audit = self.get_items_to_audit(monitor, auditor, unique_watcher_results.get(monitor.watcher.index), monitors_with_changes) + auditor.audit_these_objects(items_to_audit) + auditor.save_issues() - monitor.watcher.save() app.logger.info("Account {} is done with {}".format(account, monitor.watcher.i_am_singular)) time2 = time.time() @@ -85,6 +90,15 @@ def get_watchauditors(self, account, interval=None): mons = self.account_watchers[account] return mons + def slurp_unique_watchers(self, monitors, account, interval): + unique_watcher_results = {} + for monitor in monitors: + app.logger.info("Running {} for {} ({} minutes interval)".format(monitor.watcher.i_am_singular, account, interval)) + (items, exception_map) = monitor.watcher.slurp() + unique_watcher_results[monitor.watcher.index] = [items, exception_map] + + return unique_watcher_results + def get_alerters(self, account): """ Return a list of alerters enabled for a specific account """ return self.account_alerters[account] @@ -97,3 +111,25 @@ def get_intervals(self, account): if not interval in buckets: buckets.append(interval) return buckets + + def get_items_to_audit(self, monitor, auditor, slurped_watcher_results, monitors_with_changes): + """ + Returns the items that have changed if there are no changes in dependencies, + otherwise returns all slurped items for reauditing + """ + monitor.watcher.full_audit_list = None + if auditor.support_watcher_indexes: + for support_watcher_index in auditor.support_watcher_indexes: + if support_watcher_index in monitors_with_changes: + app.logger.debug("Upstream watcher changed {}. reauditing {}".format(support_watcher_index, monitor.watcher.index)) + monitor.watcher.full_audit_list = slurped_watcher_results[0] + if auditor.support_auditor_indexes: + for support_auditor_index in auditor.support_auditor_indexes: + if support_auditor_index in monitors_with_changes: + app.logger.debug("Upstream auditor changed {}. reauditing {}".format(support_auditor_index, monitor.watcher.index)) + monitor.watcher.full_audit_list = slurped_watcher_results[0] + + if monitor.watcher.full_audit_list: + return monitor.watcher.full_audit_list + + return [item for item in monitor.watcher.created_items + monitor.watcher.changed_items] diff --git a/security_monkey/scheduler.py b/security_monkey/scheduler.py index 96f6b8e65..9aea35627 100644 --- a/security_monkey/scheduler.py +++ b/security_monkey/scheduler.py @@ -9,11 +9,12 @@ """ from apscheduler.threadpool import ThreadPool +from apscheduler import events from apscheduler.scheduler import Scheduler from sqlalchemy.exc import OperationalError, InvalidRequestError, StatementError from security_monkey.datastore import Account, clear_old_exceptions, store_exception -from security_monkey.monitors import get_monitors +from security_monkey.monitors import get_monitors, get_monitors_and_dependencies from security_monkey.reporter import Reporter from security_monkey import app, db, jirasync @@ -36,29 +37,32 @@ def run_change_reporter(account_names, interval=None): def find_changes(accounts, monitor_names, debug=True): - monitors = get_monitors(accounts, monitor_names, debug) - for monitor in monitors: - cw = monitor.watcher - (items, exception_map) = cw.slurp() - cw.find_changes(current=items, exception_map=exception_map) - cw.save() - + """ + Runs the watcher and stores the result, reaudits all types to account + for downstream dependencies. + """ + for account_name in accounts: + monitors = get_monitors(account_name, monitor_names, debug) + for mon in monitors: + cw = mon.watcher + (items, exception_map) = cw.slurp() + cw.find_changes(current=items, exception_map=exception_map) + cw.save() audit_changes(accounts, monitor_names, False, debug) db.session.close() def audit_changes(accounts, monitor_names, send_report, debug=True): - monitors = get_monitors(accounts, monitor_names, debug) - for monitor in monitors: - _audit_changes(monitor.auditors, send_report, debug) + for account in accounts: + monitors = get_monitors_and_dependencies(account, monitor_names, debug) + for monitor in monitors: + _audit_changes(account, monitor.auditors, send_report, debug) -def _audit_changes(auditors, send_report, debug=True): +def _audit_changes(account, auditors, send_report, debug=True): """ Runs auditors on all items """ - accounts = [] try: for au in auditors: - accounts = au.accounts au.audit_all_objects() au.save_issues() if send_report: @@ -66,10 +70,10 @@ def _audit_changes(auditors, send_report, debug=True): au.email_report(report) if jirasync: - app.logger.info('Syncing {} issues on {} with Jira'.format(au.index, accounts)) - jirasync.sync_issues(accounts, au.index) + app.logger.info('Syncing {} issues on {} with Jira'.format(au.index, account)) + jirasync.sync_issues([account], au.index) except (OperationalError, InvalidRequestError, StatementError) as e: - app.logger.exception("Database error processing accounts %s, cleaning up session.", accounts) + app.logger.exception("Database error processing accounts %s, cleaning up session.", account) db.session.remove() store_exception("scheduler-audit-changes", None, e) @@ -89,9 +93,13 @@ def _clear_old_exceptions(): standalone=True, threadpool=pool, coalesce=True, - misfire_grace_time=30 + misfire_grace_time=app.config.get('MISFIRE_GRACE_TIME', 30) ) +def exception_listener(event): + store_exception("scheduler-change-reporter-uncaught", None, event.exception) + +scheduler.add_listener(exception_listener, events.EVENT_JOB_ERROR) def setup_scheduler(): """Sets up the APScheduler""" @@ -101,19 +109,22 @@ def setup_scheduler(): accounts = Account.query.filter(Account.third_party == False).filter(Account.active == True).all() # noqa accounts = [account.name for account in accounts] for account in accounts: - print "Scheduler adding account {}".format(account) + app.logger.debug("Scheduler adding account {}".format(account)) rep = Reporter(account=account) + delay = app.config.get('REPORTER_START_DELAY', 10) + for period in rep.get_intervals(account): scheduler.add_interval_job( run_change_reporter, minutes=period, - start_date=datetime.now()+timedelta(seconds=2), + start_date=datetime.now()+timedelta(seconds=delay), args=[[account], period] ) auditors = [] for monitor in rep.get_watchauditors(account): auditors.extend(monitor.auditors) - scheduler.add_cron_job(_audit_changes, hour=10, day_of_week="mon-fri", args=[auditors, True]) + scheduler.add_cron_job(_audit_changes, hour=10, day_of_week="mon-fri", args=[account, auditors, True]) + # Clear out old exceptions: scheduler.add_cron_job(_clear_old_exceptions, hour=3, minute=0) diff --git a/security_monkey/sso/views.py b/security_monkey/sso/views.py index bcc476119..aef04d5d7 100644 --- a/security_monkey/sso/views.py +++ b/security_monkey/sso/views.py @@ -9,17 +9,25 @@ import base64 import requests -from flask import Blueprint, current_app, redirect +from flask import Blueprint, current_app, redirect, request from flask.ext.restful import reqparse, Resource, Api from flask.ext.principal import Identity, identity_changed -from flask_login import login_user +from flask_security.utils import login_user + +try: + from onelogin.saml2.auth import OneLogin_Saml2_Auth + from onelogin.saml2.utils import OneLogin_Saml2_Utils + onelogin_import_success = True +except ImportError: + onelogin_import_success = False from .service import fetch_token_header_payload, get_rsa_public_key from security_monkey.datastore import User from security_monkey import db, rbac +from urlparse import urlparse mod = Blueprint('sso', __name__) api = Api(mod) @@ -230,6 +238,88 @@ def post(self): return redirect(return_to, code=302) +class OneLogin(Resource): + decorators = [rbac.allow(["anonymous"], ["GET", "POST"])] + def __init__(self): + self.reqparse = reqparse.RequestParser() + self.req = OneLogin.prepare_from_flask_request(request) + super(OneLogin, self).__init__() + + @staticmethod + def prepare_from_flask_request(req): + url_data = urlparse(req.url) + return { + 'http_host': req.host, + 'server_port': url_data.port, + 'script_name': req.path, + 'get_data': req.args.copy(), + 'post_data': req.form.copy(), + 'https': ("on" if current_app.config.get("ONELOGIN_HTTPS") else "off") + } + + def get(self): + return self.post() + + def _consumer(self, auth): + auth.process_response() + errors = auth.get_errors() + if not errors: + if auth.is_authenticated: + return True + else: + return False + else: + current_app.logger.error('Error processing %s' % (', '.join(errors))) + return False + + def post(self): + if "onelogin" not in current_app.config.get("ACTIVE_PROVIDERS"): + return "Onelogin is not enabled in the config. See the ACTIVE_PROVIDERS section.", 404 + auth = OneLogin_Saml2_Auth(self.req, current_app.config.get("ONELOGIN_SETTINGS")) + + self.reqparse.add_argument('return_to', required=False, default=current_app.config.get('WEB_PATH')) + self.reqparse.add_argument('acs', required=False) + self.reqparse.add_argument('sls', required=False) + + args = self.reqparse.parse_args() + + return_to = args['return_to'] + + if args['acs'] != None: + # valids the SAML response and checks if successfully authenticated + if self._consumer(auth): + email = auth.get_attribute(current_app.config.get("ONELOGIN_EMAIL_FIELD"))[0] + user = User.query.filter(User.email == email).first() + + # if we get an sso user create them an account + if not user: + user = User( + email=email, + active=True, + role=current_app.config.get('ONELOGIN_DEFAULT_ROLE') + # profile_picture=profile.get('thumbnailPhotoUrl') + ) + db.session.add(user) + db.session.commit() + db.session.refresh(user) + + # Tell Flask-Principal the identity changed + identity_changed.send(current_app._get_current_object(), identity=Identity(user.id)) + login_user(user) + + self_url = OneLogin_Saml2_Utils.get_self_url(self.req) + if 'RelayState' in request.form and self_url != request.form['RelayState']: + return redirect(auth.redirect_to(request.form['RelayState']), code=302) + else: + return redirect(current_app.config.get('BASE_URL'), code=302) + else: + return dict(message='OneLogin authentication failed.'), 403 + elif args['sls'] != None: + return dict(message='OneLogin SLS not implemented yet.'), 405 + else: + return redirect(auth.login(return_to=return_to)) + + class Providers(Resource): decorators = [rbac.allow(["anonymous"], ["GET"])] def __init__(self): @@ -268,6 +358,11 @@ def get(self): if google_hosted_domain is not None: google_provider['hd'] = google_hosted_domain active_providers.append(google_provider) + elif provider == "onelogin": + active_providers.append({ + 'name': 'OneLogin', + 'authorizationEndpoint': api.url_for(OneLogin) + }) else: raise Exception("Unknown authentication provider: {0}".format(provider)) @@ -277,3 +372,6 @@ def get(self): api.add_resource(Ping, '/auth/ping', endpoint='ping') api.add_resource(Google, '/auth/google', endpoint='google') api.add_resource(Providers, '/auth/providers', endpoint='providers') + +if onelogin_import_success: + api.add_resource(OneLogin, '/auth/onelogin', endpoint='onelogin') diff --git a/security_monkey/templates/security/forgot_password.html b/security_monkey/templates/security/forgot_password.html index b908f928e..1770c42e0 100644 --- a/security_monkey/templates/security/forgot_password.html +++ b/security_monkey/templates/security/forgot_password.html @@ -27,7 +27,7 @@
    - + - +
    - +