diff --git a/.gitignore b/.gitignore index 7cd8bb2f..e374abf0 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,15 @@ __pycache__/ env/ outputs/ data/ -test_*.py \ No newline at end of file +test_*.py + +# Generated application artifacts +certificate/ +certificates/ +telemetry/reports/ +telemetry/logs/ + +# macOS Finder metadata +.DS_Store +.DS_Store? +._* \ No newline at end of file diff --git a/core/cert_auth.py b/core/cert_auth.py new file mode 100644 index 00000000..19d1094d --- /dev/null +++ b/core/cert_auth.py @@ -0,0 +1,188 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Certificate helper management functions for local authentication mapping to Microsoft Graph API.""" + +import os +import logging +import datetime +from typing import Tuple +from cryptography import x509 +from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.serialization import pkcs12 + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +def setup_logging() -> None: + """Configures file logging to telemetry/logs/certificate_auth_log.txt.""" + if any(isinstance(h, logging.FileHandler) for h in logger.handlers): + return + + root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + log_dir = os.path.join(root, "telemetry", "logs") + os.makedirs(log_dir, exist_ok=True) + log_path = os.path.join(log_dir, "certificate_auth_log.txt") + + file_handler = logging.FileHandler(log_path, mode="a", encoding="utf-8") + formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") + file_handler.setFormatter(formatter) + + logger.addHandler(file_handler) + logger.propagate = True + +setup_logging() + + +def update_log_directory(tenant_id: str = None, client_id: str = None) -> None: + """Updates the log directory dynamically once tenant and client ID are known, or reverts to default.""" + # Find existing FileHandlers on the logger and remove them + for h in list(logger.handlers): + if isinstance(h, logging.FileHandler): + h.close() + logger.removeHandler(h) + + root = get_project_root() + if tenant_id and client_id: + log_dir = os.path.join(root, "telemetry", "logs", f"{tenant_id}_{client_id}") + else: + log_dir = os.path.join(root, "telemetry", "logs") + + os.makedirs(log_dir, exist_ok=True) + log_path = os.path.join(log_dir, "certificate_auth_log.txt") + + file_handler = logging.FileHandler(log_path, mode="a", encoding="utf-8") + formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") + file_handler.setFormatter(formatter) + + logger.addHandler(file_handler) + + +def get_project_root() -> str: + """Helper to locate the root path of migration-planner.""" + return os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +def get_cert_paths(cert_dir: str = "certificate", tenant_id: str = None, client_id: str = None) -> Tuple[str, str, str]: + """Returns absolute paths for cert directory, certificate.pem, and passkey.pfx.""" + root = get_project_root() + if tenant_id and client_id: + dir_path = os.path.join(root, cert_dir, f"{tenant_id}_{client_id}") + else: + dir_path = os.path.join(root, cert_dir) + pem_path = os.path.join(dir_path, "certificate.pem") + pfx_path = os.path.join(dir_path, "passkey.pfx") + return dir_path, pem_path, pfx_path + +def check_certificate_exists(cert_dir: str = "certificate", tenant_id: str = None, client_id: str = None) -> bool: + """Checks if the certificate directory exists and contains a valid cert.pfx file.""" + _, _, pfx_path = get_cert_paths(cert_dir, tenant_id, client_id) + exists = os.path.exists(pfx_path) + logger.info("Checking if certificate exists at %s: %s", pfx_path, exists) + return exists + +def generate_certificate(client_secret: str, cert_dir: str = "certificate", common_name: str = "LocalAppHybridAuth", tenant_id: str = None, client_id: str = None) -> Tuple[str, str]: + """Generates a self-signed certificate and PFX bundle using client_secret as the password. + + Returns: + Tuple containing absolute paths to the generated cert.pem and cert.pfx files. + """ + logger.info("Initializing certificate generation flow...") + dir_path, pem_path, pfx_path = get_cert_paths(cert_dir, tenant_id, client_id) + + os.makedirs(dir_path, exist_ok=True) + logger.info("Certificate directory confirmed: %s", dir_path) + + secret_bytes = client_secret.encode('utf-8') + + logger.info("Generating secure RSA private key...") + private_key = rsa.generate_private_key( + public_exponent=65537, + key_size=2048, + ) + + logger.info("Generating self-signed certificate with CN: %s...", common_name) + subject = issuer = x509.Name([ + x509.NameAttribute(NameOID.COMMON_NAME, common_name), + ]) + + cert = x509.CertificateBuilder().subject_name( + subject + ).issuer_name( + issuer + ).public_key( + private_key.public_key() + ).serial_number( + x509.random_serial_number() + ).not_valid_before( + datetime.datetime.now(datetime.timezone.utc) + ).not_valid_after( + # Valid for 2 years + datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=730) + ).sign(private_key, hashes.SHA256()) + + logger.info("Writing PEM public certificate to %s...", pem_path) + with open(pem_path, "wb") as f: + f.write(cert.public_bytes(serialization.Encoding.PEM)) + + logger.info("Serializing and writing PFX encrypted with client secret to %s...", pfx_path) + pfx_bytes = pkcs12.serialize_key_and_certificates( + name=common_name.encode('utf-8'), + key=private_key, + cert=cert, + cas=None, + encryption_algorithm=serialization.BestAvailableEncryption(secret_bytes) + ) + with open(pfx_path, "wb") as f: + f.write(pfx_bytes) + + logger.info("Certificate generation complete! Files written successfully.") + return pem_path, pfx_path + +def load_certificate(client_secret: str, cert_dir: str = "certificate", tenant_id: str = None, client_id: str = None) -> Tuple[str, str]: + """Decrypts PFX bundle using client_secret, extracting private key PEM and SHA1 thumbprint. + + Returns: + Tuple containing private_key_pem (str) and thumbprint (str). + """ + logger.info("Loading certificate files...") + _, _, pfx_path = get_cert_paths(cert_dir, tenant_id, client_id) + + if not os.path.exists(pfx_path): + raise FileNotFoundError(f"PFX certificate file not found at {pfx_path}") + + logger.info("Unlocking PFX file with client secret...") + with open(pfx_path, "rb") as f: + pfx_data = f.read() + + password_bytes = client_secret.encode('utf-8') + private_key, certificate, _ = pkcs12.load_key_and_certificates( + pfx_data, + password_bytes + ) + + logger.info("Extracting private key PEM...") + private_key_pem = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption() + ).decode("utf-8") + + logger.info("Calculating certificate SHA1 fingerprint...") + thumbprint = certificate.fingerprint(hashes.SHA1()).hex() + logger.info("Loaded certificate successfully. Thumbprint: %s", thumbprint) + + return private_key_pem, thumbprint diff --git a/core/graph/__init__.py b/core/graph/__init__.py new file mode 100644 index 00000000..3f068e81 --- /dev/null +++ b/core/graph/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unified Microsoft Graph API service layer.""" diff --git a/core/graph/client.py b/core/graph/client.py new file mode 100644 index 00000000..c61f1b47 --- /dev/null +++ b/core/graph/client.py @@ -0,0 +1,84 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unified client managing connection pools, authentication, and sessions for Microsoft Graph.""" + +import logging +from typing import List, Dict, Any, Union, Optional +import requests +from util.auth_manager import TokenManager +from util.connectors import UrlInvoker + +logger = logging.getLogger(__name__) + +class GraphClient: + """Unified client managing credentials validation and connection slots to Microsoft Graph.""" + + def __init__( + self, + tenant_id: str, + client_ids: Union[str, List[str]], + client_secrets: Union[str, List[str]], + concurrency: int = 1, + retries: int = 5, + backoff: int = 2 + ) -> None: + self.tenant_id = tenant_id + + # Normalize inputs to lists to perfectly match TokenManager parameters + self.client_ids = [client_ids] if isinstance(client_ids, str) else client_ids + self.client_secrets = [client_secrets] if isinstance(client_secrets, str) else client_secrets + + self.token_manager = TokenManager( + tenant_id=self.tenant_id, + client_ids=self.client_ids, + client_secrets=self.client_secrets, + concurrency=concurrency, + retries=retries, + backoff=backoff + ) + + self.url_invoker = UrlInvoker( + token_manager=self.token_manager, + batch_retry_count=retries, + batch_backoff=backoff, + initial_delay=1, + jitter=0.5 + ) + + def authenticate(self, required_scopes: Optional[List[str]] = None) -> None: + """Validates Entra ID scopes and fetches active tokens.""" + self.token_manager.authenticate_all(required_scopes=required_scopes) + + def get_session(self) -> requests.Session: + """Returns the unified requests Session pool.""" + return self.token_manager.get_session() + + def get_active_token(self) -> Dict[str, Any]: + """Acquires an active, refreshed token slot from TokenManager lease queue.""" + return self.token_manager.get_valid_token_slot() + + def release_token(self, token_slot: Dict[str, Any]) -> None: + """Returns token slot to the queue, completing the lease cycle.""" + self.token_manager.return_token_slot(token_slot) + + def close(self) -> None: + """Releases all connection pool and socket resources.""" + self.token_manager.close() + + def __enter__(self) -> "GraphClient": + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + self.close() diff --git a/core/graph/directory.py b/core/graph/directory.py new file mode 100644 index 00000000..0b5f190e --- /dev/null +++ b/core/graph/directory.py @@ -0,0 +1,224 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DirectoryService for querying tenant details and subscribed SKUs config.""" + +import logging +from typing import Dict, Any +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +class DirectoryService: + """Service to query Entra ID directory configuration details.""" + + def __init__(self, client: GraphClient) -> None: + self.client = client + + def get_subscribed_skus(self) -> Dict[str, Any]: + """Queries the Microsoft Graph /subscribedSkus endpoint with active retries.""" + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "ConsistencyLevel": "eventual" + } + try: + url = "https://graph.microsoft.com/v1.0/subscribedSkus" + logger.info("Querying Graph API configuration endpoint: %s", url) + resp = session.get(url, headers=headers, timeout=30.0) + resp.raise_for_status() + return resp.json() + finally: + self.client.release_token(token_slot) + + def get_tenant_primary_domain(self) -> str: + """Queries the /organization endpoint to retrieve the tenant's default or initial domain name.""" + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}" + } + try: + url = "https://graph.microsoft.com/v1.0/organization" + logger.info("Querying Graph API organization endpoint: %s", url) + resp = session.get(url, headers=headers, timeout=30.0) + resp.raise_for_status() + data = resp.json() + values = data.get("value", []) + if not values: + raise ValueError("No organization details found in Microsoft Graph.") + + domains = values[0].get("verifiedDomains", []) + + # 1. Find default domain + for d in domains: + if d.get("isDefault"): + return d.get("name") + + # 2. Fallback to initial domain (.onmicrosoft.com) + for d in domains: + if d.get("isInitial"): + return d.get("name") + + # 3. Fallback to first verified domain + if domains: + return domains[0].get("name") + + raise ValueError("No verified domains found in organization details.") + finally: + self.client.release_token(token_slot) + + def get_directory_telemetry(self, log_callback=None) -> Dict[str, Any]: + """Queries Microsoft Graph API in a single batch to fetch both domains list, user counts, and group counts.""" + logger.info("Fetching directory telemetry data using Graph API batch...") + if log_callback: + log_callback("Querying Microsoft Graph API for directory domains, users, and groups...") + + batch_requests = [ + { + "id": "domains", + "method": "GET", + "url": "/domains", + }, + { + "id": "total", + "method": "GET", + "url": "/groups?$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "security", + "method": "GET", + "url": "/groups?$filter=securityEnabled eq true and mailEnabled eq false&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "distribution", + "method": "GET", + "url": "/groups?$filter=mailEnabled eq true and securityEnabled eq false and NOT groupTypes/any(c:c eq 'Unified')&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "mail_enabled_security", + "method": "GET", + "url": "/groups?$filter=mailEnabled eq true and securityEnabled eq true and NOT groupTypes/any(c:c eq 'Unified')&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "m365", + "method": "GET", + "url": "/groups?$filter=groupTypes/any(c:c eq 'Unified')&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "dynamic", + "method": "GET", + "url": "/groups?$filter=groupTypes/any(s:s eq 'DynamicMembership')&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "users_total", + "method": "GET", + "url": "/users?$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "users_enabled", + "method": "GET", + "url": "/users?$filter=accountEnabled eq true&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "users_disabled", + "method": "GET", + "url": "/users?$filter=accountEnabled eq false&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "users_member", + "method": "GET", + "url": "/users?$filter=userType eq 'Member'&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + }, + { + "id": "users_guest", + "method": "GET", + "url": "/users?$filter=userType eq 'Guest'&$count=true&$top=1&$select=id", + "headers": {"ConsistencyLevel": "eventual"} + } + ] + + # Invoke the batch query via client's UrlInvoker + responses = self.client.url_invoker.invoke( + url="https://graph.microsoft.com/v1.0", + batch=batch_requests, + logger=log_callback or (lambda x: None), + context="DirectoryTelemetry" + ) + + domains_list = [] + counts = { + "total": 0, + "security": 0, + "distribution": 0, + "mail_enabled_security": 0, + "m365": 0, + "dynamic": 0 + } + + user_counts = { + "users_total": 0, + "users_enabled": 0, + "users_disabled": 0, + "users_member": 0, + "users_guest": 0 + } + + for resp in responses: + resp_id = resp.get("id") + if resp.get("status", 0) != 200: + error_msg = resp.get("body", {}).get("error", {}).get("message", "Unknown error") + logger.error("Failed to fetch directory telemetry for %s: status %s, message: %s", resp_id, resp.get("status"), error_msg) + raise Exception(f"Failed to fetch directory telemetry for '{resp_id}': {error_msg}") + + body = resp.get("body", {}) + if resp_id == "domains": + domains_list = body.get("value", []) + elif resp_id in counts: + count_val = body.get("@odata.count", 0) + counts[resp_id] = count_val + elif resp_id in user_counts: + count_val = body.get("@odata.count", 0) + user_counts[resp_id] = count_val + + # Normalize user counts dictionary keys for the UI + normalized_user_counts = { + "total": user_counts["users_total"], + "enabled": user_counts["users_enabled"], + "disabled": user_counts["users_disabled"], + "member": user_counts["users_member"], + "guest": user_counts["users_guest"] + } + + return { + "domains": domains_list, + "group_counts": counts, + "user_counts": normalized_user_counts + } + + + diff --git a/core/graph/reports.py b/core/graph/reports.py new file mode 100644 index 00000000..aa28e726 --- /dev/null +++ b/core/graph/reports.py @@ -0,0 +1,284 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""ReportsService encapsulating Microsoft Graph usage report generation and downloads.""" + +import os +import time +import logging +import concurrent.futures +from typing import List, Tuple +import requests +from core.graph.client import GraphClient + +import re + +def _sanitize_string(s: str) -> str: + if not s: + return "" + # Mask JWT and access tokens in query parameters or headers + s = re.sub(r'token=[^&"\')\s]+', 'token=[MASKED]', s) + s = re.sub(r'Bearer\s+[^&"\')\s]+', 'Bearer [MASKED]', s, flags=re.IGNORECASE) + return s + +logger = logging.getLogger(__name__) + + +class ReportsService: + """Service to fetch streaming M365 telemetry reports via Microsoft Graph API.""" + + def __init__(self, client: GraphClient) -> None: + self.client = client + + def download_report(self, api_url: str, output_filename: str, output_dir: str) -> None: + """Downloads a single CSV report via a streaming connection utilizing GraphClient session.""" + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}" + } + os.makedirs(output_dir, exist_ok=True) + output_path = os.path.join(output_dir, output_filename) + + try: + logger.info("Calling Graph report endpoint for %s...", output_filename) + resp = session.get(api_url, headers=headers, allow_redirects=False, stream=True) + + # Graph APIs return status code 302 redirect to a pre-authenticated S3/Azure Blob URL + if resp.status_code == 302: + resp.close() + location_url = resp.headers.get("Location") + if not location_url: + raise ConnectionError(f"302 redirect returned for {output_filename} but Location header is missing.") + + logger.info("Pre-authenticated storage redirect retrieved. Waiting 2 seconds before download...") + time.sleep(2) + + max_retries = 5 + retry_interval = 30 + for attempt in range(1, max_retries + 1): + try: + logger.info("[Attempt %d/%d] Downloading report stream to %s...", attempt, max_retries, output_filename) + with requests.get(location_url, stream=True) as csv_response: + csv_response.raise_for_status() + with open(output_path, "wb") as f: + for chunk in csv_response.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + break + except requests.exceptions.RequestException as error: + if attempt < max_retries: + logger.warning("[Attempt %d/%d] Failed stream download. Retrying in %ds... (Error: %s)", attempt, max_retries, retry_interval, _sanitize_string(str(error))) + time.sleep(retry_interval) + else: + logger.error("Failed downloading %s after %d attempts. Error: %s", output_filename, max_retries, _sanitize_string(str(error))) + raise ConnectionError(f"Failed downloading report after {max_retries} attempts.") + + logger.info("Success! Saved report to: %s", output_path) + + elif resp.status_code == 200: + logger.info("Unexpected 200 OK status returned. Saving direct streaming stream...") + with open(output_path, "wb") as f: + for chunk in resp.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + resp.close() + logger.info("Success! Saved report to: %s", output_path) + else: + resp.close() + logger.error("Graph report request failed with status code %d: %s", resp.status_code, _sanitize_string(resp.text)) + raise ConnectionError(f"Microsoft Graph API request failed with status code {resp.status_code}") + finally: + self.client.release_token(token_slot) + + def download_reports_batch(self, reports: List[Tuple[str, str]], output_dir: str) -> None: + """Downloads a list of reports concurrently using thread executors.""" + logger.info("Starting parallel batch download for %d reports...", len(reports)) + + with concurrent.futures.ThreadPoolExecutor(max_workers=len(reports)) as executor: + futures = [ + executor.submit(self.download_report, url, filename, output_dir) + for url, filename in reports + ] + # Gather all futures, raising exceptions if any thread failed + for future in concurrent.futures.as_completed(futures): + future.result() + + def download_o365_active_user_detail(self, output_dir: str) -> None: + """Downloads the Office 365 active user details CSV report (180 days).""" + url = "https://graph.microsoft.com/v1.0/reports/getOffice365ActiveUserDetail(period='D180')" + self.download_report(url, "Office365ActiveUserDetail(180d).csv", output_dir) + + def download_o365_active_user_counts(self, output_dir: str) -> None: + """Downloads the Office 365 30-day active user counts CSV report.""" + url = "https://graph.microsoft.com/v1.0/reports/getOffice365ActiveUserCounts(period='D30')" + self.download_report(url, "Office365ActiveUserCounts(30d).csv", output_dir) + + def download_m365_app_details(self, output_dir: str) -> None: + """Downloads both M365 App user details and counts CSV reports concurrently.""" + reports = [ + ("https://graph.microsoft.com/v1.0/reports/getM365AppUserDetail(period='D180')", "M365AppUserDetail(180d).csv"), + ("https://graph.microsoft.com/v1.0/reports/getM365AppUserCounts(period='D180')", "getM365AppUserCounts(180d).csv") + ] + self.download_reports_batch(reports, output_dir) + + def download_sharepoint_onedrive_details(self, output_dir: str) -> None: + """Downloads SharePoint site usage, OneDrive account usage, OneDrive activity, and M365 App details CSV reports concurrently.""" + reports = [ + ("https://graph.microsoft.com/v1.0/reports/getSharePointSiteUsageDetail(period='D180')", "SharePointSiteUsageDetail(180d).csv"), + ("https://graph.microsoft.com/v1.0/reports/getOneDriveUsageAccountDetail(period='D180')", "OneDriveUsageAccountDetail(180d).csv"), + ("https://graph.microsoft.com/v1.0/reports/getOneDriveActivityUserDetail(period='D180')", "OneDriveActivityUserDetail(180d).csv"), + ("https://graph.microsoft.com/v1.0/reports/getM365AppUserDetail(period='D180')", "M365AppUserDetail_sp_od(180d).csv") + ] + self.download_reports_batch(reports, output_dir) + + def download_sharepoint_details(self, output_dir: str) -> None: + """Downloads only SharePoint site usage detail report.""" + self.download_report("https://graph.microsoft.com/v1.0/reports/getSharePointSiteUsageDetail(period='D180')", "SharePointSiteUsageDetail(180d).csv", output_dir) + + def download_onedrive_details(self, output_dir: str) -> None: + """Downloads OneDrive usage account, activity, and app user detail reports concurrently.""" + reports = [ + ("https://graph.microsoft.com/v1.0/reports/getOneDriveUsageAccountDetail(period='D180')", "OneDriveUsageAccountDetail(180d).csv"), + ("https://graph.microsoft.com/v1.0/reports/getOneDriveActivityUserDetail(period='D180')", "OneDriveActivityUserDetail(180d).csv"), + ("https://graph.microsoft.com/v1.0/reports/getM365AppUserDetail(period='D180')", "M365AppUserDetail_sp_od(180d).csv") + ] + self.download_reports_batch(reports, output_dir) + + def download_mailbox_usage_detail(self, output_dir: str) -> None: + """Downloads Exchange mailbox usage detail CSV report (180 days).""" + self.download_report("https://graph.microsoft.com/v1.0/reports/getMailboxUsageDetail(period='D180')", "MailboxUsageDetail(180d).csv", output_dir) + + def download_email_app_usage_detail(self, output_dir: str) -> None: + """Downloads Exchange email app usage detail CSV report (180 days).""" + self.download_report("https://graph.microsoft.com/v1.0/reports/getEmailAppUsageUserDetail(period='D180')", "EmailAppUsageUserDetail(180d).csv", output_dir) + + def search_cloud_pst_files(self) -> dict: + """Queries Microsoft Graph Search API to locate cloud-stored PST archive files across all active regions in a paginated fashion, up to a total of 2000 files.""" + url = "https://graph.microsoft.com/v1.0/search/query" + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json", + "Content-Type": "application/json" + } + + total_hits = [] + regions = ["NAM", "EUR", "APC"] + page_size = 500 + max_total_limit = 2000 + + try: + for region in regions: + # Calculate how many hits we have already fetched in total + current_fetched = 0 + for r_resp in total_hits: + for hc in r_resp.get("hitsContainers", []): + current_fetched += len(hc.get("hits", [])) + + remaining_limit = max_total_limit - current_fetched + if remaining_limit <= 0: + logger.info(f"Reached total limit of {max_total_limit} files. Stopping search across regions.") + break + + offset = 0 + has_more = True + region_response = None + region_hits_container = None + + while has_more: + # Request only up to the remaining allowed limit + current_page_size = min(page_size, remaining_limit) + + payload = { + "requests": [ + { + "entityTypes": ["driveItem"], + "query": {"queryString": "fileextension:pst"}, + "from": offset, + "size": current_page_size, + "region": region + } + ] + } + logger.info(f"Executing Graph Search query for cloud PST archives in region: {region} (offset: {offset}, limit: {current_page_size})...") + resp = session.post(url, json=payload, headers=headers) + if resp.status_code == 200: + data = resp.json() + page_response_list = data.get("value", []) + + if not page_response_list: + has_more = False + continue + + page_response = page_response_list[0] + page_containers = page_response.get("hitsContainers", []) + + if not page_containers: + has_more = False + continue + + container = page_containers[0] + hits = container.get("hits", []) + total_count = container.get("total", 0) + more_results = container.get("moreResultsAvailable", False) + + if region_response is None: + region_response = { + "@odata.type": page_response.get("@odata.type"), + "searchTerms": page_response.get("searchTerms", []), + "hitsContainers": [ + { + "@odata.type": container.get("@odata.type"), + "total": total_count, + "moreResultsAvailable": False, + "hits": [] + } + ] + } + region_hits_container = region_response["hitsContainers"][0] + + region_hits_container["hits"].extend(hits) + + region_fetched = len(region_hits_container["hits"]) + if region_fetched >= remaining_limit: + logger.info(f"Reached remaining limit of {remaining_limit} files in region {region}. Stopping paginated fetch.") + region_hits_container["hits"] = region_hits_container["hits"][:remaining_limit] + has_more = False + elif more_results and len(hits) > 0: + offset += page_size + else: + has_more = False + + elif resp.status_code == 400 and "Only valid regions are" in resp.text: + logger.info(f"Region {region} is not active for this tenant. Skipping.") + has_more = False + else: + raise ConnectionError(f"Graph Search failed for region {region} (HTTP {resp.status_code}): {resp.text}") + + if region_response is not None: + total_hits.append(region_response) + + return {"value": total_hits} + finally: + self.client.release_token(token_slot) + + def download_email_app_usage_apps_user_counts(self, output_dir: str) -> None: + """Downloads Exchange email app usage apps user counts CSV report (180 days).""" + self.download_report("https://graph.microsoft.com/v1.0/reports/getEmailAppUsageAppsUserCounts(period='D180')", "EmailAppUsageAppsUserCounts(180d).csv", output_dir) + + diff --git a/core/graph/security.py b/core/graph/security.py new file mode 100644 index 00000000..b4d3be03 --- /dev/null +++ b/core/graph/security.py @@ -0,0 +1,139 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SecurityService encapsulating Microsoft Graph security and governance policy queries.""" + +import logging +from core.graph.client import GraphClient + +logger = logging.getLogger(__name__) + +class SecurityService: + """Service to interact with M365 Security and Information Protection configurations.""" + + def __init__(self, client: GraphClient) -> None: + self.client = client + + def fetch_sensitivity_labels(self) -> list[dict]: + """Fetches the sensitivity labels configured for the tenant in JSON format.""" + url = "https://graph.microsoft.com/v1.0/security/dataSecurityAndGovernance/sensitivityLabels" + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + try: + logger.info("Querying Microsoft Graph information protection sensitivity labels...") + resp = session.get(url, headers=headers) + if resp.status_code == 200: + data = resp.json() + return data.get("value", []) + else: + logger.error("Graph sensitivityLabels endpoint failed with status %d: %s", resp.status_code, resp.text) + raise ConnectionError(f"Microsoft Graph API request failed with status {resp.status_code}") + finally: + self.client.release_token(token_slot) + + def fetch_conditional_access_policies(self) -> list[dict]: + """Fetches the Microsoft Entra Conditional Access policies configured for the tenant.""" + url = "https://graph.microsoft.com/v1.0/identity/conditionalAccess/policies" + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + try: + logger.info("Querying Entra ID Conditional Access policies...") + resp = session.get(url, headers=headers) + if resp.status_code == 200: + data = resp.json() + return data.get("value", []) + elif resp.status_code in [401, 403]: + logger.error("Conditional Access endpoint permission error: %d %s", resp.status_code, resp.text) + raise PermissionError("Policy.Read.All or Policy.Read permission required.") + else: + logger.error("Conditional Access endpoint failed with status %d: %s", resp.status_code, resp.text) + raise ConnectionError(f"Microsoft Graph API request failed with status {resp.status_code}") + finally: + self.client.release_token(token_slot) + + def fetch_sso_service_principals(self) -> list[dict]: + """Fetches Microsoft Entra Enterprise Applications (Service Principals) to analyze Single Sign-On.""" + url = "https://graph.microsoft.com/v1.0/servicePrincipals?$select=id,appDisplayName,preferredSingleSignOnMode&$top=100" + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json" + } + try: + logger.info("Querying Entra ID Service Principals for Single Sign-On modes...") + resp = session.get(url, headers=headers) + if resp.status_code == 200: + data = resp.json() + return data.get("value", []) + elif resp.status_code in [401, 403]: + logger.error("Service Principals endpoint permission error: %d %s", resp.status_code, resp.text) + raise PermissionError("Application.Read.All permission required.") + else: + logger.error("Service Principals endpoint failed with status %d: %s", resp.status_code, resp.text) + raise ConnectionError(f"Microsoft Graph API request failed with status {resp.status_code}") + finally: + self.client.release_token(token_slot) + + def search_cloud_pst_files(self) -> dict: + """Queries Microsoft Graph Search API to locate cloud-stored PST archive files.""" + url = "https://graph.microsoft.com/v1.0/search/query" + token_slot = self.client.get_active_token() + session = self.client.get_session() + + headers = { + "Authorization": f"Bearer {token_slot['token']}", + "Accept": "application/json", + "Content-Type": "application/json" + } + payload = { + "requests": [ + { + "entityTypes": ["driveItem"], + "query": {"queryString": "fileextension:pst"}, + "from": 0, + "size": 50 + } + ] + } + try: + logger.info("Executing Graph Search query for cloud PST archives...") + resp = session.post(url, json=payload, headers=headers) + if resp.status_code == 200: + return resp.json() + elif resp.status_code in [401, 403]: + logger.warning("Graph Search permission error: %d %s", resp.status_code, resp.text) + return {} + else: + logger.warning("Graph Search query failed with status %d: %s", resp.status_code, resp.text) + return {} + except Exception as e: + logger.warning("Exception during Graph Search query: %s", e) + return {} + finally: + self.client.release_token(token_slot) + + + diff --git a/core/powershell/__init__.py b/core/powershell/__init__.py new file mode 100644 index 00000000..41d440bb --- /dev/null +++ b/core/powershell/__init__.py @@ -0,0 +1 @@ +# Init powershell core package diff --git a/core/powershell/calendar.py b/core/powershell/calendar.py new file mode 100644 index 00000000..d32a0624 --- /dev/null +++ b/core/powershell/calendar.py @@ -0,0 +1,52 @@ +import json +from core.powershell.client import PowerShellClient + +class CalendarStatsService: + def __init__(self, client: PowerShellClient): + self.client = client + + def fetch_calendar_attachments_policy(self) -> dict: + """Locates certificate, triggers powershell execution, and parses Exchange calendar configurations.""" + try: + cert_path = self.client.locate_certificate() + except Exception as e: + raise RuntimeError(f"Failed to locate certificate for authentication: {str(e)}") + + args = [ + "-AppId", self.client.client_id, + "-Organization", self.client.tenant_id, + "-CertificatePath", cert_path + ] + if self.client.cert_password: + args += ["-CertificatePassword", self.client.cert_password] + + raw_output = self.client.execute_script("scripts/exchange_calendar_metadata.ps1", args) + + if not raw_output or not raw_output.strip(): + return { + "RoomsCount": 0, + "RoomsError": None, + "RoomsNaming": None, + "EquipmentCount": 0, + "EquipmentError": None, + "CanShareAttachments": True, + "OwaPolicyError": None, + "OrganizationApps": [], + "AppsError": None + } + + # Parse JSON block in case of warning/header lines outputted by powershell environment + lines = raw_output.strip().split('\n') + json_str = "" + for line in lines: + stripped = line.strip() + if stripped.startswith("[") or stripped.startswith("{") or json_str: + json_str += stripped + + if json_str: + try: + return json.loads(json_str) + except json.JSONDecodeError as e: + raise RuntimeError(f"Failed to decode JSON from script output: {str(e)}. Output: {json_str}") + + raise RuntimeError(f"PowerShell returned non-JSON format: {raw_output}") diff --git a/core/powershell/client.py b/core/powershell/client.py new file mode 100644 index 00000000..3d91da4f --- /dev/null +++ b/core/powershell/client.py @@ -0,0 +1,44 @@ +import os +import subprocess +import logging + +logger = logging.getLogger("PowerShellClient") + +class PowerShellClient: + def __init__(self, tenant_id, client_id, client_secret, cert_tenant_id=None): + self.tenant_id = tenant_id + self.client_id = client_id + self.cert_password = client_secret + self.cert_tenant_id = cert_tenant_id or tenant_id + + def locate_certificate(self) -> str: + """Locates the automated hybrid auth PFX certificate path.""" + from core.cert_auth import get_cert_paths + _, _, pfx_path = get_cert_paths(tenant_id=self.cert_tenant_id, client_id=self.client_id) + if not os.path.exists(pfx_path): + raise FileNotFoundError(f"Hybrid auth PFX certificate not found at {pfx_path}. Please complete the hybrid authentication flow on the login page first.") + return pfx_path + + def execute_script(self, script_relative_path: str, args: list) -> str: + """Executes pwsh with the specified script and arguments.""" + script_path = os.path.join(os.path.dirname(__file__), script_relative_path) + + if not os.path.exists(script_path): + raise FileNotFoundError(f"PowerShell script not found at {script_path}") + + # Construct CLI command + command = ["pwsh", "-NoProfile", "-NonInteractive", "-File", script_path] + args + + try: + result = subprocess.run( + command, + capture_output=True, + text=True, + check=True + ) + return result.stdout + except subprocess.CalledProcessError as e: + logger.error(f"PowerShell script failed: {e.stderr}") + raise RuntimeError(f"PowerShell script execution failed: {e.stderr or e.stdout}") + except FileNotFoundError: + raise RuntimeError("PowerShell core ('pwsh') is not installed or not in PATH. Please install it (e.g. 'brew install powershell' on macOS).") diff --git a/core/powershell/exchange_connectors.py b/core/powershell/exchange_connectors.py new file mode 100644 index 00000000..9748c903 --- /dev/null +++ b/core/powershell/exchange_connectors.py @@ -0,0 +1,46 @@ +import json +import logging +from core.powershell.client import PowerShellClient + +logger = logging.getLogger("ExchangeConnectorsService") + +class ExchangeConnectorsService: + def __init__(self, client: PowerShellClient): + self.client = client + + def fetch_exchange_connectors(self) -> dict: + """Executes get_connectors.ps1 and parses the JSON response.""" + logger.info("Executing Exchange Connectors PowerShell script...") + try: + cert_path = self.client.locate_certificate() + except FileNotFoundError as e: + logger.error(str(e)) + raise RuntimeError(str(e)) + + args = [ + "-AppId", self.client.client_id, + "-Organization", self.client.tenant_id, + "-CertificatePath", cert_path + ] + + if self.client.cert_password: + args.extend(["-CertificatePassword", self.client.cert_password]) + + try: + output = self.client.execute_script("scripts/get_connectors.ps1", args) + + # Extract JSON from output + json_str = output + if "{" in output: + json_str = output[output.find("{"):] + + if not json_str.strip(): + return {"InboundConnectors": [], "OutboundConnectors": [], "Errors": {}} + + return json.loads(json_str) + except json.JSONDecodeError as e: + logger.error(f"Failed to parse Exchange Connectors JSON: {e}") + raise RuntimeError(f"Failed to parse Exchange Connectors data: {e}") + except Exception as e: + logger.error(f"Exchange Connectors retrieval failed: {e}") + raise RuntimeError(f"Exchange Connectors retrieval failed: {e}") diff --git a/core/powershell/mailbox.py b/core/powershell/mailbox.py new file mode 100644 index 00000000..e2e543b0 --- /dev/null +++ b/core/powershell/mailbox.py @@ -0,0 +1,39 @@ +import json +from core.powershell.client import PowerShellClient + +class MailboxStatsService: + def __init__(self, client: PowerShellClient): + self.client = client + + def fetch_mailbox_and_folder_stats(self) -> dict: + """Locates certificate, triggers powershell execution, and parses shared mailbox and public folder stats.""" + try: + cert_path = self.client.locate_certificate() + except Exception as e: + raise RuntimeError(f"Failed to locate certificate for authentication: {str(e)}") + + args = [ + "-AppId", self.client.client_id, + "-Organization", self.client.tenant_id, + "-CertificatePath", cert_path + ] + if self.client.cert_password: + args += ["-CertificatePassword", self.client.cert_password] + + raw_output = self.client.execute_script("scripts/get_mailbox_and_folder_stats.ps1", args) + + if not raw_output or not raw_output.strip(): + return {} + + try: + return json.loads(raw_output) + except json.JSONDecodeError: + # Parse JSON block in case of warning/header lines outputted by powershell environment + lines = raw_output.strip().split('\n') + json_str = "" + for line in lines: + if line.startswith("[") or line.startswith("{") or json_str: + json_str += line + if json_str: + return json.loads(json_str) + raise RuntimeError(f"PowerShell returned non-JSON format: {raw_output}") diff --git a/core/powershell/retention.py b/core/powershell/retention.py new file mode 100644 index 00000000..214a6ae2 --- /dev/null +++ b/core/powershell/retention.py @@ -0,0 +1,39 @@ +import json +from core.powershell.client import PowerShellClient + +class RetentionService: + def __init__(self, client: PowerShellClient): + self.client = client + + def fetch_retention_policies(self) -> list: + """Locates certificate, triggers powershell execution, and parses retention policies.""" + try: + cert_path = self.client.locate_certificate() + except Exception as e: + raise RuntimeError(f"Failed to locate certificate for authentication: {str(e)}") + + args = [ + "-AppId", self.client.client_id, + "-Organization", self.client.tenant_id, + "-CertificatePath", cert_path + ] + if self.client.cert_password: + args += ["-CertificatePassword", self.client.cert_password] + + raw_output = self.client.execute_script("scripts/get_retention_policies.ps1", args) + + if not raw_output or not raw_output.strip(): + return [] + + try: + return json.loads(raw_output) + except json.JSONDecodeError: + # Parse JSON block in case of warning/header lines outputted by powershell environment + lines = raw_output.strip().split('\n') + json_str = "" + for line in lines: + if line.startswith("[") or line.startswith("{") or json_str: + json_str += line + if json_str: + return json.loads(json_str) + raise RuntimeError(f"PowerShell returned non-JSON format: {raw_output}") diff --git a/core/powershell/scripts/exchange_calendar_metadata.ps1 b/core/powershell/scripts/exchange_calendar_metadata.ps1 new file mode 100644 index 00000000..f71f3ab6 --- /dev/null +++ b/core/powershell/scripts/exchange_calendar_metadata.ps1 @@ -0,0 +1,93 @@ +param( + [Parameter(Mandatory=$true)] + [string]$AppId, + [Parameter(Mandatory=$true)] + [string]$Organization, + [Parameter(Mandatory=$true)] + [string]$CertificatePath, + [Parameter(Mandatory=$false)] + [string]$CertificatePassword +) + +$ErrorActionPreference = "Stop" + +if (-not (Get-Module -ListAvailable -Name ExchangeOnlineManagement)) { + throw "ExchangeOnlineManagement PowerShell module is not installed." +} + +Import-Module ExchangeOnlineManagement + +# Connect to Exchange Online using App-Only Cert Auth +if ($CertificatePassword) { + $secPassword = ConvertTo-SecureString -String $CertificatePassword -AsPlainText -Force + Connect-ExchangeOnline -CertificateFilePath $CertificatePath -CertificatePassword $secPassword -AppId $AppId -Organization $Organization -ShowBanner:$false -WarningAction SilentlyContinue +} else { + Connect-ExchangeOnline -CertificateFilePath $CertificatePath -AppId $AppId -Organization $Organization -ShowBanner:$false -WarningAction SilentlyContinue +} + +try { + $roomsCount = 0 + $roomsNaming = $null + $roomsError = $null + try { + $rooms = @(Get-Mailbox -RecipientTypeDetails RoomMailbox -ResultSize Unlimited) + $roomsCount = $rooms.Count + $roomsNaming = if ($rooms) { (($rooms | Select-Object -First 5 -ExpandProperty Name) -join ", ") } else { $null } + } catch { + $roomsError = $_.Exception.Message + } + + $equipmentCount = 0 + $equipmentError = $null + try { + $equipment = @(Get-Mailbox -RecipientTypeDetails EquipmentMailbox -ResultSize Unlimited) + $equipmentCount = $equipment.Count + } catch { + $equipmentError = $_.Exception.Message + } + + $owaPolicyError = $null + $canShareAttachments = $true + try { + $owaPolicy = Get-OwaMailboxPolicy | Where-Object { $_.IsDefault -eq $true } + if (-not $owaPolicy) { + $owaPolicy = Get-OwaMailboxPolicy | Select-Object -First 1 + } + $canShareAttachments = if ($owaPolicy) { $owaPolicy.ClassicAttachmentsEnabled } else { $true } + } catch { + $owaPolicyError = $_.Exception.Message + } + + $orgAppsData = @() + $appsError = $null + try { + $orgApps = @(Get-App -OrganizationApp -ErrorAction Stop) + if ($orgApps) { + foreach ($app in $orgApps) { + $orgAppsData += [PSCustomObject]@{ + DisplayName = $app.DisplayName + AppId = $app.AppId + Enabled = $app.Enabled + } + } + } + } catch { + $appsError = $_.Exception.Message + } + + $result = [PSCustomObject]@{ + RoomsCount = $roomsCount + RoomsError = $roomsError + RoomsNaming = $roomsNaming + EquipmentCount = $equipmentCount + EquipmentError = $equipmentError + CanShareAttachments = $canShareAttachments + OwaPolicyError = $owaPolicyError + OrganizationApps = $orgAppsData + AppsError = $appsError + } + $result | ConvertTo-Json +} +finally { + Disconnect-ExchangeOnline -Confirm:$false -ErrorAction SilentlyContinue +} diff --git a/core/powershell/scripts/get_connectors.ps1 b/core/powershell/scripts/get_connectors.ps1 new file mode 100644 index 00000000..fd76e3af --- /dev/null +++ b/core/powershell/scripts/get_connectors.ps1 @@ -0,0 +1,76 @@ +param( + [Parameter(Mandatory=$true)] + [string]$AppId, + [Parameter(Mandatory=$true)] + [string]$Organization, + [Parameter(Mandatory=$true)] + [string]$CertificatePath, + [Parameter(Mandatory=$false)] + [string]$CertificatePassword +) + +$ErrorActionPreference = "Stop" + +if (-not (Get-Module -ListAvailable -Name ExchangeOnlineManagement)) { + throw "ExchangeOnlineManagement PowerShell module is not installed. Please install it beforehand by running: Install-Module -Name ExchangeOnlineManagement -Scope CurrentUser" +} + +Import-Module ExchangeOnlineManagement + +# Connect to Exchange Online using App-Only Cert Auth +if ($CertificatePassword) { + $secPassword = ConvertTo-SecureString -String $CertificatePassword -AsPlainText -Force + Connect-ExchangeOnline -CertificateFilePath $CertificatePath -CertificatePassword $secPassword -AppId $AppId -Organization $Organization -ShowBanner:$false -WarningAction SilentlyContinue +} else { + Connect-ExchangeOnline -CertificateFilePath $CertificatePath -AppId $AppId -Organization $Organization -ShowBanner:$false -WarningAction SilentlyContinue +} + +try { + $errors = @{} + $inbound = @() + $outbound = @() + + try { + $inboundRaw = Get-InboundConnector -ErrorAction Stop + if ($inboundRaw) { + foreach ($conn in @($inboundRaw)) { + $inbound += @{ + Name = $conn.Identity + Enabled = $conn.Enabled + ConnectorType = $conn.ConnectorType + SenderDomains = ($conn.SenderDomains -join ", ") + RequireTls = $conn.RequireTls + } + } + } + } catch { + $errors["InboundConnectors"] = $_.Exception.Message + } + + try { + $outboundRaw = Get-OutboundConnector -ErrorAction Stop + if ($outboundRaw) { + foreach ($conn in @($outboundRaw)) { + $outbound += @{ + Name = $conn.Identity + Enabled = $conn.Enabled + RecipientDomains = ($conn.RecipientDomains -join ", ") + SmartHosts = ($conn.SmartHosts -join ", ") + UseMxRecord = $conn.UseMxRecord + } + } + } + } catch { + $errors["OutboundConnectors"] = $_.Exception.Message + } + + $result = [PSCustomObject]@{ + InboundConnectors = $inbound + OutboundConnectors = $outbound + Errors = $errors + } + $result | ConvertTo-Json -Depth 5 +} +finally { + Disconnect-ExchangeOnline -Confirm:$false -ErrorAction SilentlyContinue +} diff --git a/core/powershell/scripts/get_mailbox_and_folder_stats.ps1 b/core/powershell/scripts/get_mailbox_and_folder_stats.ps1 new file mode 100644 index 00000000..689c3916 --- /dev/null +++ b/core/powershell/scripts/get_mailbox_and_folder_stats.ps1 @@ -0,0 +1,131 @@ +param( + [Parameter(Mandatory=$true)] + [string]$AppId, + [Parameter(Mandatory=$true)] + [string]$Organization, + [Parameter(Mandatory=$true)] + [string]$CertificatePath, + [Parameter(Mandatory=$false)] + [string]$CertificatePassword +) + +$ErrorActionPreference = "Stop" + +# Check if ExchangeOnlineManagement module is installed beforehand +if (-not (Get-Module -ListAvailable -Name ExchangeOnlineManagement)) { + throw "ExchangeOnlineManagement PowerShell module is not installed. Please install it beforehand by running: Install-Module -Name ExchangeOnlineManagement -Scope CurrentUser" +} + +Import-Module ExchangeOnlineManagement + +# Connect to Exchange Online using App-Only Cert Auth +if ($CertificatePassword) { + $secPassword = ConvertTo-SecureString -String $CertificatePassword -AsPlainText -Force + Connect-ExchangeOnline -CertificateFilePath $CertificatePath -CertificatePassword $secPassword -AppId $AppId -Organization $Organization -ShowBanner:$false -WarningAction SilentlyContinue +} else { + Connect-ExchangeOnline -CertificateFilePath $CertificatePath -AppId $AppId -Organization $Organization -ShowBanner:$false -WarningAction SilentlyContinue +} + +try { + $errors = @{} + + # 1. Query Shared Mailboxes + $sharedCount = $null + $sharedTotalBytes = $null + try { + $sharedMailboxes = Get-Mailbox -RecipientTypeDetails SharedMailbox -ResultSize Unlimited -ErrorAction Stop + if ($sharedMailboxes) { + $sharedStats = @($sharedMailboxes | Get-MailboxStatistics -ErrorAction Stop) + $sharedCount = @($sharedMailboxes).Count + $sharedTotalBytes = [long]0 + foreach ($stat in $sharedStats) { + if ($stat.TotalItemSize -and $stat.TotalItemSize.Value) { + try { + $bytes = $stat.TotalItemSize.Value.ToBytes() + } catch { + $bytesStr = $stat.TotalItemSize.ToString() + if ($bytesStr -match '\(([\d,]+) bytes\)') { + $bytes = [long]($Matches[1] -replace ',', '') + } else { + $bytes = [long]0 + } + } + $sharedTotalBytes += $bytes + } + } + } else { + $sharedCount = 0 + $sharedTotalBytes = 0 + } + } catch { + $errors["SharedMailboxes"] = $_.Exception.Message + } + + # 2. Query Public Folders + $pfCount = $null + try { + $pfs = Get-PublicFolder -Recurse -ResultSize Unlimited -ErrorAction Stop + if ($pfs) { + $pfCount = @($pfs).Count + } else { + $pfCount = 0 + } + } catch { + $errors["PublicFolders"] = $_.Exception.Message + } + + # 2b. Query Mail-Enabled Public Folders + $mailPfCount = $null + try { + $mailPfs = Get-MailPublicFolder -ResultSize Unlimited -ErrorAction Stop + if ($mailPfs) { + $mailPfCount = @($mailPfs).Count + } else { + $mailPfCount = 0 + } + } catch { + $errors["MailPublicFolders"] = $_.Exception.Message + } + + # 2c. Query Public Folder Stats (Size) + $pfTotalBytes = $null + try { + $pfStats = Get-PublicFolderStatistics -ResultSize Unlimited -ErrorAction Stop + if ($pfStats) { + $pfTotalBytes = [long]0 + foreach ($stat in @($pfStats)) { + if ($stat.TotalItemSize -and $stat.TotalItemSize.Value) { + try { + $bytes = $stat.TotalItemSize.Value.ToBytes() + } catch { + $bytesStr = $stat.TotalItemSize.ToString() + if ($bytesStr -match '\(([\d,]+) bytes\)') { + $bytes = [long]($Matches[1] -replace ',', '') + } else { + $bytes = [long]0 + } + } + $pfTotalBytes += $bytes + } + } + } else { + $pfTotalBytes = 0 + } + } catch { + $errors["PublicFolderStats"] = $_.Exception.Message + } + + # Output results as JSON + $result = [PSCustomObject]@{ + SharedMailboxesCount = $sharedCount + SharedMailboxesTotalBytes = $sharedTotalBytes + PublicFoldersCount = $pfCount + PublicFoldersTotalBytes = $pfTotalBytes + MailPublicFoldersCount = $mailPfCount + Errors = $errors + } + $result | ConvertTo-Json +} +finally { + Disconnect-ExchangeOnline -Confirm:$false -ErrorAction SilentlyContinue +} diff --git a/core/powershell/scripts/get_retention_policies.ps1 b/core/powershell/scripts/get_retention_policies.ps1 new file mode 100644 index 00000000..b151b147 --- /dev/null +++ b/core/powershell/scripts/get_retention_policies.ps1 @@ -0,0 +1,102 @@ +param( + [Parameter(Mandatory=$true)] + [string]$AppId, + [Parameter(Mandatory=$true)] + [string]$Organization, + [Parameter(Mandatory=$true)] + [string]$CertificatePath, + [Parameter(Mandatory=$false)] + [string]$CertificatePassword +) + +$ErrorActionPreference = "Stop" + +# Check if ExchangeOnlineManagement module is installed beforehand +if (-not (Get-Module -ListAvailable -Name ExchangeOnlineManagement)) { + throw "ExchangeOnlineManagement PowerShell module is not installed. Please install it beforehand by running: Install-Module -Name ExchangeOnlineManagement -Scope CurrentUser" +} + +Import-Module ExchangeOnlineManagement + +# Connect to Security & Compliance PowerShell using App-Only Cert Auth +if ($CertificatePassword) { + $secPassword = ConvertTo-SecureString -String $CertificatePassword -AsPlainText -Force + Connect-IPPSSession -CertificateFilePath $CertificatePath -CertificatePassword $secPassword -AppId $AppId -Organization $Organization -ShowBanner:$false -WarningAction SilentlyContinue +} else { + Connect-IPPSSession -CertificateFilePath $CertificatePath -AppId $AppId -Organization $Organization -ShowBanner:$false -WarningAction SilentlyContinue +} + +try { + # Retrieve policies + $policies = Get-RetentionCompliancePolicy + $output = @() + + if ($policies) { + # Retrieve all compliance rules at once + $allRules = Get-RetentionComplianceRule + $ruleMap = @{} + if ($allRules) { + foreach ($rule in @($allRules)) { + if ($rule.Policy) { + $rawKey = $rule.Policy.ToString() + $ruleMap[$rawKey] = $rule + + # Extract the policy name if the reference is a DistinguishedName + if ($rawKey -match "CN=([^,]+)") { + $cnKey = $Matches[1] + $ruleMap[$cnKey] = $rule + } + } + } + } + + # Handle case where $policies is not an array + $policies_list = @($policies) + + foreach ($policy in $policies_list) { + $duration = "N/A" + $action = "N/A" + $trigger = "N/A" + + $rule = $null + if ($ruleMap.ContainsKey($policy.Name)) { + $rule = $ruleMap[$policy.Name] + } elseif ($ruleMap.ContainsKey($policy.Identity.ToString())) { + $rule = $ruleMap[$policy.Identity.ToString()] + } elseif ($policy.Guid -and $ruleMap.ContainsKey($policy.Guid.ToString())) { + $rule = $ruleMap[$policy.Guid.ToString()] + } + + if ($rule) { + $duration = $rule.RetentionDuration + $action = $rule.RetentionAction + $trigger = $rule.RetentionTrigger + } + + # Construct a combined custom object + $policyObj = [PSCustomObject]@{ + Name = $policy.Name + Comment = $policy.Comment + Workload = $policy.Workload + Mode = $policy.Mode + DistributionStatus = $policy.DistributionStatus + Enabled = $policy.Enabled + Duration = $duration + RetentionAction = $action + RetentionTrigger = $trigger + Identity = $policy.Identity.ToString() + WhenCreated = $policy.WhenCreated + WhenChanged = $policy.WhenChanged + CreatedBy = $policy.CreatedBy + LastModifiedBy = $policy.LastModifiedBy + } + $output += $policyObj + } + $output | ConvertTo-Json -Depth 5 + } else { + "[]" + } +} +finally { + Disconnect-ExchangeOnline -Confirm:$false +} diff --git a/deal_assistant.py b/deal_assistant.py new file mode 100644 index 00000000..044c1879 --- /dev/null +++ b/deal_assistant.py @@ -0,0 +1,1665 @@ +# Copyright 2026 Google LLC +"""Standalone application for the License Usage and Telemetry view.""" + +import os +import pandas as pd +import customtkinter as ctk +from telemetry.m365_telemetry import M365TelemetryTab, async_logger +import logging +from telemetry.power_automate import PowerAutomateScanner +from telemetry.styles import * +from telemetry.user_persona_analysis import run_user_persona_pipeline + +import queue +import threading +import tkinter as tk +from tkinter import filedialog, messagebox +import ui.exchange_online_ui +import ui.chats_ui +import ui.files_ui + +# Create custom helper base class to embed CTk windows as CTkFrames +class EmbeddedCTkFrameHelper(ctk.CTkFrame): + _current_master = None + + def __init__(self, master=None, *args, **kwargs): + actual_master = getattr(EmbeddedCTkFrameHelper, "_current_master", master) + super().__init__(actual_master, *args, **kwargs) + + def title(self, *args, **kwargs): + pass + + def geometry(self, *args, **kwargs): + pass + + def protocol(self, *args, **kwargs): + pass + + def attributes(self, *args, **kwargs): + pass + + +# Patch estimator tool base classes in-memory +ui.exchange_online_ui.MigrationEstimatorTool.__bases__ = (EmbeddedCTkFrameHelper,) +ui.chats_ui.ChatMigrationEstimatorTool.__bases__ = (EmbeddedCTkFrameHelper,) + + +class EmbeddedExchangeOnlineTool(ui.exchange_online_ui.MigrationEstimatorTool): + def __init__(self, master, controller, *args, **kwargs): + EmbeddedCTkFrameHelper._current_master = master + self.controller = controller + super().__init__() + + # Populate credentials + self.tenant_id.set(self.controller.stored_tenant) + self.client_ids.set(self.controller.stored_client) + self.client_secrets.set(self.controller.stored_secret) + + def create_entry(self, parent, label, var, show=None): + if label in ["Tenant ID", "Client ID", "Client Secret"]: + if parent.winfo_exists(): + parent.pack_forget() + if parent.master and parent.master.winfo_exists(): + parent.master.pack_forget() + return + super().create_entry(parent, label, var, show) + + def go_back_to_selector(self): + if hasattr(self, "_back_callback") and self._back_callback: + self._back_callback() + + +class EmbeddedChatTool(ui.chats_ui.ChatMigrationEstimatorTool): + def __init__(self, master, controller, *args, **kwargs): + EmbeddedCTkFrameHelper._current_master = master + self.controller = controller + super().__init__() + + # Populate credentials + self.tenant_id.set(self.controller.stored_tenant) + self.client_ids.set(self.controller.stored_client) + self.client_secrets.set(self.controller.stored_secret) + + def create_entry(self, parent, label, var, show=None): + if label in ["Tenant ID", "Client ID", "Client Secret"]: + if parent.winfo_exists(): + parent.pack_forget() + if parent.master and parent.master.winfo_exists(): + parent.master.pack_forget() + return + super().create_entry(parent, label, var, show) + + def go_back_to_selector(self): + if hasattr(self, "_back_callback") and self._back_callback: + self._back_callback() + + +class EmbeddedFilesTool(ui.files_ui.FileMigrationEstimatorTool): + def __init__(self, master, controller, *args, **kwargs): + EmbeddedCTkFrameHelper._current_master = master + self.controller = controller + super().__init__() + + # Populate credentials + self.tenant_id.set(self.controller.stored_tenant) + self.client_ids.set(self.controller.stored_client) + self.client_secrets.set(self.controller.stored_secret) + + def create_entry(self, parent, label, var, show=None): + if label in ["Tenant ID", "Client ID", "Client Secret"]: + if parent.winfo_exists(): + parent.pack_forget() + if parent.master and parent.master.winfo_exists(): + parent.master.pack_forget() + return + super().create_entry(parent, label, var, show) + + def go_back_to_selector(self): + if hasattr(self, "_back_callback") and self._back_callback: + self._back_callback() + + +class MigrationPlannerView(ctk.CTkFrame): + """Container for the Migration Planner workload selector and tool views.""" + + def __init__(self, master, controller, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.controller = controller + + # Workload selector frame + self.selector_frame = ctk.CTkFrame(self, fg_color="transparent") + self.selector_frame.pack(fill="both", expand=True) + + self.setup_selector_ui() + + # Active tool frame container + self.active_tool_frame = None + + def setup_selector_ui(self): + # Title of selector + ctk.CTkLabel( + self.selector_frame, + text="Select Workload to Estimate", + font=FONT_HEADER_MEDIUM, + text_color=COLOR_TEXT_MAIN, + ).pack(pady=(40, 10), anchor="w", padx=40) + + ctk.CTkLabel( + self.selector_frame, + text="Plan your migration by estimating data size and timelines for your workloads.", + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_SUB, + ).pack(pady=(0, 30), anchor="w", padx=40) + + # Card container grid + self.cards_container = ctk.CTkFrame(self.selector_frame, fg_color="transparent") + self.cards_container.pack(fill="x", padx=40) + + self.cards_container.grid_columnconfigure((0, 1, 2), weight=1, uniform="equal") + + # Card 1: Exchange + self.create_workload_card( + parent=self.cards_container, + column=0, + icon="📩", + title="Exchange Online", + desc="Estimate migration time and resource sizes for mailboxes, calendars, and contacts.", + callback=lambda: self.launch_tool("Exchange") + ) + + # Card 2: Chat + self.create_workload_card( + parent=self.cards_container, + column=1, + icon="💬", + title="Chat (Teams)", + desc="Plan Teams private chats, group channels, and message history migration.", + callback=lambda: self.launch_tool("Chat") + ) + + # Card 3: Files + self.create_workload_card( + parent=self.cards_container, + column=2, + icon="📁", + title="Files (SharePoint/OneDrive)", + desc="Analyze OneDrive personal sites and SharePoint team site collections.", + callback=lambda: self.launch_tool("Files") + ) + + def create_workload_card(self, parent, column, icon, title, desc, callback): + card = ctk.CTkFrame( + parent, + fg_color=COLOR_SURFACE, + corner_radius=12, + border_width=1, + border_color=COLOR_OUTLINE_LIGHT + ) + card.grid(row=0, column=column, padx=10, pady=10, sticky="nsew") + + # Icon + ctk.CTkLabel( + card, + text=icon, + font=ctk.CTkFont(size=36), + text_color=COLOR_PRIMARY + ).pack(pady=(25, 10)) + + # Title + ctk.CTkLabel( + card, + text=title, + font=FONT_BODY_BOLD, + text_color=COLOR_TEXT_MAIN + ).pack(pady=(0, 10)) + + # Description + ctk.CTkLabel( + card, + text=desc, + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_SUB, + wraplength=220, + justify="center" + ).pack(fill="both", expand=True, padx=20, pady=(0, 20)) + + # Launch Button + btn = ctk.CTkButton( + card, + text="Open Planner", + command=callback, + height=36, + corner_radius=8, + fg_color=COLOR_PRIMARY, + hover_color=COLOR_PRIMARY_HOVER, + font=FONT_BODY_BOLD + ) + btn.pack(pady=(0, 25), padx=20, fill="x") + + def launch_tool(self, workload): + # Hide selector + self.selector_frame.pack_forget() + + # Clean old tool if any + if self.active_tool_frame: + self.active_tool_frame.destroy() + self.active_tool_frame = None + + # Instantiate the embedded tool + if workload == "Exchange": + self.active_tool_frame = EmbeddedExchangeOnlineTool(self, self.controller) + elif workload == "Chat": + self.active_tool_frame = EmbeddedChatTool(self, self.controller) + elif workload == "Files": + self.active_tool_frame = EmbeddedFilesTool(self, self.controller) + + if self.active_tool_frame: + self.active_tool_frame._back_callback = self.show_selector + self.active_tool_frame.pack(fill="both", expand=True) + + def show_selector(self): + if self.active_tool_frame: + self.active_tool_frame.pack_forget() + self.active_tool_frame.destroy() + self.active_tool_frame = None + self.selector_frame.pack(fill="both", expand=True) + + +class UserPersonaAnalysisView(ctk.CTkFrame): + """Container for the User Persona Analysis screen.""" + + def __init__(self, master, controller, **kwargs): + super().__init__(master, fg_color="transparent", **kwargs) + self.controller = controller + self.personas_list = [] + self.output_csv = None + self.results_frame = None + + # Main container + self.container = ctk.CTkFrame(self, fg_color="transparent") + self.container.pack(fill="both", expand=True) + + self.setup_ui() + + def setup_ui(self): + + # API Key Form Card (Top aligned, matches header width) + self.form_card = ctk.CTkFrame( + self.container, + fg_color=COLOR_SURFACE, + corner_radius=12, + border_width=1, + border_color=COLOR_OUTLINE_LIGHT + ) + self.form_card.pack(fill="x", side="top", pady=(5, 15)) + + self.form_container = ctk.CTkFrame(self.form_card, fg_color="transparent") + self.form_container.pack(fill="x", expand=True, padx=40, pady=25) + + # API Key Input Label + self.api_key_lbl = ctk.CTkLabel( + self.form_container, + text="Gemini API Key", + font=FONT_BODY_BOLD, + text_color=COLOR_TEXT_MAIN + ) + self.api_key_lbl.pack(anchor="w", pady=(10, 5)) + + # API Key Input Wrapper Frame (No hardcoded width for responsiveness) + self.api_key_border = tk.Frame( + self.form_container, height=42, + highlightbackground=COLOR_OUTLINE, highlightcolor=COLOR_PRIMARY, highlightthickness=1, + bd=0, background=COLOR_SURFACE + ) + self.api_key_border.pack(fill="x", expand=True, pady=(0, 15)) + self.api_key_border.pack_propagate(False) + + self.api_key_entry = ctk.CTkEntry( + self.api_key_border, border_width=0, fg_color="transparent", text_color=COLOR_TEXT_MAIN, + placeholder_text="Enter Gemini API Key", show="*" + ) + self.api_key_entry.pack(fill="both", expand=True, padx=10, pady=2) + + # Strategy Selector Heading + self.strategy_lbl = ctk.CTkLabel( + self.form_container, + text="Analysis Strategy", + font=FONT_BODY_BOLD, + text_color=COLOR_TEXT_MAIN + ) + self.strategy_lbl.pack(anchor="w", pady=(5, 5)) + + self.strategy_var = tk.StringVar(value="heuristic") + + self.strategy_frame = ctk.CTkFrame(self.form_container, fg_color="transparent") + self.strategy_frame.pack(fill="x", pady=(0, 20)) + + # Radio option 1: AI Heuristic matching + self.radio_heuristic = ctk.CTkRadioButton( + self.strategy_frame, + text="Strategy 1: Direct LLM Classification", + variable=self.strategy_var, + value="heuristic", + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_MAIN, + fg_color=COLOR_PRIMARY, + hover_color=COLOR_SECONDARY_HOVER + ) + self.radio_heuristic.pack(anchor="w", pady=5) + + # Radio option 2: K-Means + LLM Summary + self.radio_kmeans = ctk.CTkRadioButton( + self.strategy_frame, + text="Strategy 2: K-Means pre-clustering & LLM summary", + variable=self.strategy_var, + value="kmeans", + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_MAIN, + fg_color=COLOR_PRIMARY, + hover_color=COLOR_SECONDARY_HOVER + ) + self.radio_kmeans.pack(anchor="w", pady=5) + + # Generate Button + self.generate_btn = ctk.CTkButton( + self.form_container, + text="Generate User Personas", + command=self.on_generate_clicked, + width=250, + height=40, + corner_radius=20, + fg_color=COLOR_PRIMARY, + hover_color=COLOR_PRIMARY_HOVER, + font=FONT_BODY_BOLD + ) + self.generate_btn.pack(pady=(0, 10)) + + # Status loading label (hidden by default) + self.status_lbl = ctk.CTkLabel( + self.form_container, + text="", + font=FONT_BODY_MEDIUM, + text_color=COLOR_PRIMARY, + justify="center", + wraplength=700 + ) + + # Disclaimer Box always packed at bottom of self.container (No hardcoded width, center-aligned) + self.disclaimer_card = ctk.CTkFrame( + self.container, + fg_color="#FFF9E6", + corner_radius=8, + border_width=1, + border_color="#FFE0B2" + ) + self.disclaimer_card.pack(fill="x", side="bottom", pady=20, padx=40) + + self.disclaimer_lbl = ctk.CTkLabel( + self.disclaimer_card, + text="⚠️ Disclaimer: This tool uses Generative AI (GenAI) to generate user personas and may not be fully accurate.\nIt should be used for a rough estimate and not as hard evidence.", + font=FONT_BODY_MEDIUM, + text_color="#B78103", + justify="center", + anchor="center" + ) + self.disclaimer_lbl.pack(fill="both", expand=True, padx=20, pady=15) + + def set_loading_state(self, is_loading): + state = "disabled" if is_loading else "normal" + self.generate_btn.configure(state=state) + self.api_key_entry.configure(state=state) + self.radio_heuristic.configure(state=state) + self.radio_kmeans.configure(state=state) + + if is_loading: + self.status_lbl.configure(text_color=COLOR_PRIMARY) + self.status_lbl.pack(pady=10) + if self.results_frame: + self.results_frame.pack_forget() + + def update_status(self, step): + if step == "Fetching Reports": + msg = "⏳ Fetching Reports..." + elif step == "Aggregating Data": + msg = "⏳ Aggregating Data..." + elif step == "Generating insights using Gemini": + msg = "⏳ Generating insights using Gemini..." + else: + msg = f"⏳ {step}..." + + self.controller.after(0, lambda: self.status_lbl.configure(text=msg, text_color=COLOR_PRIMARY)) + + def on_pipeline_success(self, result): + self.set_loading_state(False) + self.status_lbl.pack_forget() # Hide the status label on success + + self.personas_list = result.get("personas", []) + self.output_csv = result.get("dataset_path") + + # Hide the input settings card + self.form_card.pack_forget() + + if self.results_frame: + self.results_frame.pack_forget() + self.results_frame.destroy() + + self.render_persona_results() + + def on_pipeline_error(self, err_msg): + self.set_loading_state(False) + self.status_lbl.configure( + text=f"✖ Failed to generate dataset: {err_msg}", + text_color=COLOR_ERROR + ) + self.status_lbl.pack(pady=10) + + def show_regenerate_form(self): + if self.results_frame: + self.results_frame.pack_forget() + self.status_lbl.pack_forget() + self.form_card.pack(fill="x", side="top", pady=(5, 15)) + + def render_persona_results(self): + if not self.personas_list or not self.output_csv: + return + + # 1. Create a container frame for results + self.results_frame = ctk.CTkFrame(self.container, fg_color="transparent") + # Pack below the form card, above the disclaimer + self.results_frame.pack(fill="both", expand=True, padx=40, pady=(0, 10)) + + # Title / Export Header row + header_row = ctk.CTkFrame(self.results_frame, fg_color="transparent") + header_row.pack(fill="x", pady=(0, 10)) + + ctk.CTkLabel( + header_row, + text="Generated User Personas", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN + ).pack(side="left") + + # Export Button + btn_export = ctk.CTkButton( + header_row, + text="Export Assignments CSV", + command=self.export_assignments_csv, + width=180, + height=32, + corner_radius=16, + font=FONT_BODY_BOLD, + fg_color="transparent", + border_width=1, + border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, + hover_color=COLOR_SECONDARY_HOVER + ) + btn_export.pack(side="right") + + # Regenerate Button + btn_regenerate = ctk.CTkButton( + header_row, + text="Regenerate Response", + command=self.show_regenerate_form, + width=180, + height=32, + corner_radius=16, + font=FONT_BODY_BOLD, + fg_color="transparent", + border_width=1, + border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, + hover_color=COLOR_SECONDARY_HOVER + ) + btn_regenerate.pack(side="right", padx=(0, 10)) + + # Export status label (empty initially) + self.export_status_lbl = ctk.CTkLabel( + header_row, + text="", + font=FONT_BODY_MEDIUM, + text_color=COLOR_SUCCESS + ) + self.export_status_lbl.pack(side="right", padx=(0, 15)) + + # 2. Get user counts per persona from output CSV + try: + df = pd.read_csv(self.output_csv) + counts = df['Assigned_Persona_ID'].value_counts().to_dict() + except Exception as e: + logger.error(f"Failed to read user counts: {e}") + counts = {} + + # 3. Create a scrollable frame for cards if there are many + cards_scroll = ctk.CTkScrollableFrame( + self.results_frame, + fg_color="transparent", + height=400 + ) + cards_scroll.pack(fill="both", expand=True) + + # 4. Generate Persona Cards + for persona in self.personas_list: + p_id = persona.get("id") + p_title = persona.get("title", "Persona") + p_emoji = persona.get("emoji", "👤") + p_desc = persona.get("description", "") + p_patterns = persona.get("behavior_patterns", []) + p_count = counts.get(p_id, 0) + + # Card Container + card = ctk.CTkFrame( + cards_scroll, + fg_color=COLOR_SURFACE, + border_width=1, + border_color=COLOR_OUTLINE_LIGHT, + corner_radius=8 + ) + card.pack(fill="x", pady=6, padx=2) + + # Card Header (Title & User Count) + card_header = ctk.CTkFrame(card, fg_color="transparent") + card_header.pack(fill="x", padx=15, pady=(12, 6)) + + ctk.CTkLabel( + card_header, + text=f"{p_emoji} {p_title}", + font=FONT_HEADER_SMALL, + text_color=COLOR_PRIMARY + ).pack(side="left") + + ctk.CTkLabel( + card_header, + text=f"{p_count} users assigned", + font=FONT_BODY_BOLD, + text_color=COLOR_TEXT_SUB + ).pack(side="right") + + # Card Description + ctk.CTkLabel( + card, + text=p_desc, + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_MAIN, + justify="left", + anchor="w", + wraplength=1050 + ).pack(fill="x", padx=15, pady=(0, 10)) + + # Behavior Patterns Bullet points + if p_patterns: + patterns_frame = ctk.CTkFrame(card, fg_color="transparent") + patterns_frame.pack(fill="x", padx=15, pady=(0, 12)) + + for pattern in p_patterns: + ctk.CTkLabel( + patterns_frame, + text=f"• {pattern}", + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_MAIN, + justify="left", + anchor="w", + wraplength=1030 + ).pack(fill="x", pady=1) + + def export_assignments_csv(self): + if not self.output_csv: + return + + file_path = filedialog.asksaveasfilename( + defaultextension=".csv", + filetypes=[("CSV files", "*.csv"), ("All files", "*.*")], + title="Export User Persona Assignments", + initialfile="M365_User_Persona_Assignments.csv", + parent=self + ) + if not file_path: + return + + try: + df = pd.read_csv(self.output_csv) + export_cols = ['User Principal Name', 'App_Access_Profile', 'Assigned_Persona_ID', 'Assigned_Persona_Title'] + df_export = df[[c for c in export_cols if c in df.columns]] + df_export.to_csv(file_path, index=False, encoding="utf-8-sig") + self.export_status_lbl.configure( + text="✔ Exported successfully!", + text_color=COLOR_SUCCESS + ) + except Exception as e: + self.export_status_lbl.configure( + text=f"✖ Export failed: {e}", + text_color=COLOR_ERROR + ) + + def on_generate_clicked(self): + tenant = self.controller.stored_tenant + client = self.controller.stored_client + secret = self.controller.stored_secret + + if not tenant or not client or not secret: + self.status_lbl.configure( + text="✖ Error: Please connect your Azure App Credentials on the welcome screen first.", + text_color=COLOR_ERROR + ) + self.status_lbl.pack(pady=10) + return + + api_key = self.api_key_entry.get().strip() + if not api_key: + self.status_lbl.configure( + text="✖ Warning: Please enter a valid Gemini API Key.", + text_color=COLOR_ERROR + ) + self.status_lbl.pack(pady=10) + return + + # Start loading state + self.set_loading_state(True) + self.update_status("Starting pipeline") + + # Run pipeline in a background thread + def run_thread(): + try: + # Set output path + output_dir = os.path.join("telemetry", "reports", f"{tenant}_{client}") + output_csv = os.path.join(output_dir, "user_activity_data.csv") + + # Execute pipeline + res = run_user_persona_pipeline( + tenant_id=tenant, + client_id=client, + client_secret=secret, + gemini_api_key=api_key, + output_csv_path=output_csv, + strategy=self.strategy_var.get(), + status_callback=lambda step: self.update_status(step) + ) + + # Success + self.controller.after(0, lambda: self.on_pipeline_success(res)) + except Exception as e: + logger.exception("Error generating user persona dataset") + self.controller.after(0, lambda err=str(e): self.on_pipeline_error(err)) + + threading.Thread(target=run_thread, daemon=True).start() + + + + + +# Orchestrator logging +logger = logging.getLogger("M365TelemetryAsyncLogger.TelemetryOrchestrator") + + + +class TelemetryApp(ctk.CTk): + """Standalone application for the License Usage and Telemetry view.""" + + def __init__(self): + super().__init__() + logger.info("Initializing TelemetryApp application...") + self.title("Deal Assistant") # CITATION: self.title("Deal Assistant") + self.geometry("1230x950") # Expanded window width to support increased sidebar dimensions + + # FIX: Bind the window close button to a custom exit handler + # to prevent CustomTkinter 'after script' errors when closing. + self.protocol("WM_DELETE_WINDOW", self.on_closing) # CITATION: self.protocol("WM_DELETE_WINDOW", self.on_closing) + + # Initialize variables required by the M365TelemetryTab + self.retries = ctk.IntVar(value=30) # CITATION: self.retries = ctk.IntVar(value=30) + self.backoff = ctk.IntVar(value=2) # CITATION: self.backoff = ctk.IntVar(value=2) + + # Stage 1: In-memory variables to store connection credentials + self.stored_tenant = "" + self.stored_client = "" + self.stored_secret = "" + + # Page containers + self.auth_frame = ctk.CTkFrame(self, fg_color="transparent") + self.report_frame = ctk.CTkFrame(self, fg_color="transparent") + + # Render Page 1 (Authentication screen) + self.setup_auth_ui() + + # Render Page 2 (Reports Dashboard with Sidebar) + self.setup_report_ui() + + # Initial view + self.show_auth_page() + logger.info("TelemetryApp UI initialized successfully.") + + def setup_auth_ui(self): + """Builds a modern, polished Connection interface for Page 1.""" + # Welcome Branding Card + self.brand_card = ctk.CTkFrame( + self.auth_frame, + fg_color=COLOR_SURFACE, + corner_radius=12, + border_width=1, + border_color=COLOR_OUTLINE_LIGHT + ) + self.brand_card.pack(fill="x", padx=40, pady=(60, 20)) + + self.brand_title = ctk.CTkLabel( + self.brand_card, + text="Deal Assistant", + font=FONT_HEADER_MEDIUM, + text_color=COLOR_PRIMARY + ) + self.brand_title.pack(pady=(25, 5)) + + self.brand_subtitle = ctk.CTkLabel( + self.brand_card, + text="Connect your Azure App Credentials to begin auditing your tenant.", + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_SUB + ) + self.brand_subtitle.pack(pady=(0, 25)) + + # Credentials Form Box + self.credentials_card = ctk.CTkFrame( + self.auth_frame, + fg_color=COLOR_SURFACE, + corner_radius=12, + border_width=1, + border_color=COLOR_OUTLINE_LIGHT + ) + self.credentials_card.pack(fill="both", expand=True, padx=40, pady=(0, 40)) + + self.form_container = ctk.CTkFrame(self.credentials_card, fg_color="transparent") + self.form_container.pack(pady=40, anchor="center") + + # Tenant ID Input + self.tenant_lbl = ctk.CTkLabel(self.form_container, text="Tenant ID", font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN) + self.tenant_lbl.pack(anchor="w", pady=(10, 5)) + import tkinter as tk + + # Tenant ID Input Wrapper Frame (using native tk.Frame to ensure full X11 border rendering) + self.tenant_border = tk.Frame( + self.form_container, width=850, height=42, + highlightbackground=COLOR_OUTLINE, highlightcolor=COLOR_PRIMARY, highlightthickness=1, + bd=0, background=COLOR_SURFACE + ) + self.tenant_border.pack(pady=(0, 15)) + self.tenant_border.pack_propagate(False) + + self.tenant_entry = ctk.CTkEntry( + self.tenant_border, border_width=0, fg_color="transparent", text_color=COLOR_TEXT_MAIN, + placeholder_text="Enter Tenant ID" + ) + self.tenant_entry.pack(fill="both", expand=True, padx=10, pady=2) + + # Client ID Input + self.client_lbl = ctk.CTkLabel(self.form_container, text="Client ID", font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN) + self.client_lbl.pack(anchor="w", pady=(10, 5)) + + self.client_border = tk.Frame( + self.form_container, width=850, height=42, + highlightbackground=COLOR_OUTLINE, highlightcolor=COLOR_PRIMARY, highlightthickness=1, + bd=0, background=COLOR_SURFACE + ) + self.client_border.pack(pady=(0, 15)) + self.client_border.pack_propagate(False) + + self.client_entry = ctk.CTkEntry( + self.client_border, border_width=0, fg_color="transparent", text_color=COLOR_TEXT_MAIN, + placeholder_text="Enter Client ID" + ) + self.client_entry.pack(fill="both", expand=True, padx=10, pady=2) + + # Client Secret Input + self.secret_lbl = ctk.CTkLabel(self.form_container, text="Client Secret", font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN) + self.secret_lbl.pack(anchor="w", pady=(10, 5)) + + self.secret_border = tk.Frame( + self.form_container, width=850, height=42, + highlightbackground=COLOR_OUTLINE, highlightcolor=COLOR_PRIMARY, highlightthickness=1, + bd=0, background=COLOR_SURFACE + ) + self.secret_border.pack(pady=(0, 30)) + self.secret_border.pack_propagate(False) + + self.secret_entry = ctk.CTkEntry( + self.secret_border, show="*", border_width=0, fg_color="transparent", text_color=COLOR_TEXT_MAIN, + placeholder_text="Enter Client Secret" + ) + self.secret_entry.pack(fill="both", expand=True, padx=10, pady=2) + + # Status & Feedback Display + self.auth_status_lbl = ctk.CTkLabel(self.form_container, text="", font=FONT_BODY_MEDIUM, text_color=COLOR_ERROR) + self.auth_status_lbl.pack(pady=(0, 15)) + + # Action Submission Trigger + self.connect_btn = ctk.CTkButton( + self.form_container, + text="Connect & Continue", + command=self.on_connect_clicked, + height=44, + corner_radius=8, + fg_color=COLOR_PRIMARY, + hover_color=COLOR_PRIMARY_HOVER, + font=FONT_BODY_BOLD + ) + self.connect_btn.pack(fill="x", pady=(0, 10)) + + + def setup_report_ui(self): + """Instantiates the ReportsPage frame which contains the collapsible navigation structure.""" + self.reports_page = ReportsPage( + master=self.report_frame, + controller=self, + retries_var=self.retries, + backoff_var=self.backoff + ) + self.reports_page.pack(fill="both", expand=True) + + def on_connect_clicked(self): + """Validates inputs, caches credentials in-memory, checks certificate status, and transitions/generates cert.""" + tenant = self.tenant_entry.get().strip() + client = self.client_entry.get().strip() + secret = self.secret_entry.get().strip() + + logger.info("Connect & Continue clicked. Verifying connection credentials...") + if not tenant or not client or not secret: + logger.warning("Connection failed: Missing one or more required credential fields.") + self.auth_status_lbl.configure(text="Error: Tenant ID, Client ID, and Client Secret are required.", text_color="red") + return + + self.auth_status_lbl.configure(text="") + + # Cache connection details safely in memory + self.stored_tenant = tenant + self.stored_client = client + self.stored_secret = secret + + logger.info("Credentials validated and cached in memory. Updating log directories...") + + # Update log directory to use sub-folder based on tenant and client + from telemetry.m365_telemetry import update_log_directory as update_license_log_dir + from core.cert_auth import update_log_directory as update_cert_log_dir + + update_license_log_dir(tenant, client) + update_cert_log_dir(tenant, client) + + from core.cert_auth import check_certificate_exists, generate_certificate, load_certificate + + if check_certificate_exists(tenant_id=tenant, client_id=client): + try: + # Decrypt the PFX certificate using the client secret + load_certificate(secret, tenant_id=tenant, client_id=client) + except Exception as e: + from tkinter import messagebox + messagebox.showerror( + "Certificate Decryption Error", + f"Unable to unlock certificate with Client Secret. Proceeding with standard Client Secret authentication fallback.\n\nError: {e}", + parent=self + ) + # Proceed to reports page in either case + self.show_reports_page() + else: + try: + # Generate new certificate and pfx encrypted with the client secret + pem_path, _ = generate_certificate(secret, tenant_id=tenant, client_id=client) + # Setup instructions UI requesting the user to upload it to Entra + self.setup_cert_instructions_ui(pem_path) + except Exception as e: + from tkinter import messagebox + messagebox.showerror( + "Certificate Generation Error", + f"Unable to generate certificate. Proceeding with standard Client Secret authentication fallback.\n\nError: {e}", + parent=self + ) + self.show_reports_page() + + def setup_cert_instructions_ui(self, pem_path): + """Displays certificate upload instructions screen when a new certificate is generated.""" + self.form_container.pack_forget() + + if hasattr(self, "cert_container") and self.cert_container: + self.cert_container.destroy() + + self.cert_container = ctk.CTkFrame(self.credentials_card, fg_color="transparent") + self.cert_container.pack(pady=30, padx=50, fill="both", expand=True) + + ctk.CTkLabel( + self.cert_container, + text="Certificate Upload", + font=FONT_HEADER_MEDIUM, + text_color=COLOR_PRIMARY + ).pack(anchor="w", pady=(0, 15)) + + intro_text = ( + "A new security certificate has been generated for hybrid authentication.\n\n" + "Uploading this certificate is highly recommended, but optional:" + ) + self.cert_intro_lbl = ctk.CTkLabel( + self.cert_container, + text=intro_text, + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_MAIN, + justify="left", + wraplength=1000 + ) + self.cert_intro_lbl.pack(anchor="w", pady=(0, 10)) + + upload_statement = ( + "• If you UPLOAD the certificate:\n" + " All report sections will be fully functional." + ) + self.cert_upload_lbl = ctk.CTkLabel( + self.cert_container, + text=upload_statement, + font=FONT_BODY_BOLD, + text_color=COLOR_SUCCESS, + justify="left", + wraplength=1000 + ) + self.cert_upload_lbl.pack(anchor="w", pady=(0, 10)) + + skip_statement = ( + "• If you SKIP uploading the certificate:\n" + " You can still run the reports. However, sections relying on certificate-based authentication (such as detailed Calendar settings, " + "Shared/Public mailbox statistics, Retention Policies etc.) will be skipped and show as unavailable." + ) + self.cert_skip_lbl = ctk.CTkLabel( + self.cert_container, + text=skip_statement, + font=FONT_BODY_BOLD, + text_color=COLOR_ERROR, + justify="left", + wraplength=1000 + ) + self.cert_skip_lbl.pack(anchor="w", pady=(0, 10)) + + # Prominent Configuration Callout Card + self.cert_info_card = ctk.CTkFrame( + self.cert_container, + fg_color=COLOR_TONAL_BG, + border_width=1, + border_color=COLOR_OUTLINE_LIGHT, + corner_radius=8 + ) + self.cert_info_card.pack(fill="x", pady=(15, 25)) + + # Title of info card + self.cert_card_title = ctk.CTkLabel( + self.cert_info_card, + text="Upload Instructions", + font=FONT_BODY_BOLD, + text_color=COLOR_TONAL_TEXT, + justify="left" + ) + self.cert_card_title.pack(anchor="w", padx=15, pady=(15, 5)) + + # Instructions body + self.cert_footer_lbl = ctk.CTkLabel( + self.cert_info_card, + text=f"1. Locate the certificate file generated at:\n {pem_path}\n\n2. Log in to the Microsoft Azure portal and navigate to the App Registration with Client ID:\n {self.stored_client}\n\n3. Upload the certificate under:\n Certificates & secrets -> Certificates -> Upload certificate", + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_MAIN, + justify="left", + wraplength=970 + ) + self.cert_footer_lbl.pack(anchor="w", padx=15, pady=(0, 15)) + + self.cert_continue_btn = ctk.CTkButton( + self.cert_container, + text="Continue", + command=self.on_cert_continue_clicked, + height=40, + corner_radius=20, + font=FONT_BODY_BOLD, + fg_color=COLOR_PRIMARY, + hover_color=COLOR_PRIMARY_HOVER + ) + self.cert_continue_btn.pack(fill="x") + + # Bind resize event to dynamically adjust text wrapping based on container width + self.cert_container.bind( + "", + lambda event: self.adjust_cert_wraplengths( + event.width - 20 + ) + ) + + def adjust_cert_wraplengths(self, width): + w = max(200, width) + if hasattr(self, "cert_intro_lbl") and self.cert_intro_lbl.winfo_exists(): + self.cert_intro_lbl.configure(wraplength=w) + if hasattr(self, "cert_upload_lbl") and self.cert_upload_lbl.winfo_exists(): + self.cert_upload_lbl.configure(wraplength=w) + if hasattr(self, "cert_skip_lbl") and self.cert_skip_lbl.winfo_exists(): + self.cert_skip_lbl.configure(wraplength=w) + if hasattr(self, "cert_card_title") and self.cert_card_title.winfo_exists(): + self.cert_card_title.configure(wraplength=w - 30) + if hasattr(self, "cert_footer_lbl") and self.cert_footer_lbl.winfo_exists(): + self.cert_footer_lbl.configure(wraplength=w - 30) + + def on_cert_continue_clicked(self): + """Validates certificate after user claims to have uploaded it and transitions to reports.""" + from core.cert_auth import load_certificate + try: + # Verify we can unlock/read the cert successfully + load_certificate(self.stored_secret, tenant_id=self.stored_tenant, client_id=self.stored_client) + except Exception as e: + from tkinter import messagebox + messagebox.showerror( + "Certificate Verification Error", + f"Unable to verify certificate. Proceeding with standard Client Secret authentication fallback.\n\nError: {e}", + parent=self + ) + self.show_reports_page() + + # Reset UI form in case of subsequent logins + if hasattr(self, "cert_container") and self.cert_container: + self.cert_container.pack_forget() + self.form_container.pack(pady=40, padx=50, fill="both", expand=True) + + def on_disconnect_clicked(self): + """Clears all session properties and safely resets screens back to Page 1.""" + logger.info("Disconnect clicked. Clearing cached session and credentials...") + # Wipes local entry buffers + self.tenant_entry.delete(0, "end") + self.client_entry.delete(0, "end") + self.secret_entry.delete(0, "end") + self.auth_status_lbl.configure(text="") + + # Wipes stored in-memory configurations + self.stored_tenant = "" + self.stored_client = "" + self.stored_secret = "" + + # Clears variables inside nested dashboards + self.reports_page.clear_session_data() + + # Revert log directories to default + from telemetry.m365_telemetry import update_log_directory as update_license_log_dir + from core.cert_auth import update_log_directory as update_cert_log_dir + + update_license_log_dir() + update_cert_log_dir() + + # Clean up cert screen UI and restore normal entry layout + if hasattr(self, "cert_container") and self.cert_container: + try: + self.cert_container.pack_forget() + except Exception: + pass + self.form_container.pack(pady=40, padx=50, fill="both", expand=True) + + # Shifts screen orientation + self.show_auth_page() + logger.info("Session successfully disconnected. Returned to Auth page.") + + def show_auth_page(self): + """Transitions view port to Page 1 (Authentication screen).""" + logger.info("Showing Authentication Page.") + self.report_frame.pack_forget() + self.auth_frame.pack(fill="both", expand=True) + + def show_reports_page(self): + """Transitions view port to Page 2 (Reports Dashboard).""" + logger.info("Showing Reports Dashboard Page.") + self.auth_frame.pack_forget() + self.report_frame.pack(fill="both", expand=True) + + def log_msg(self, text): # CITATION: def log_msg(self, text): + """Simple callback handler for telemetry UI logs. Pipes to log file instead of stdout.""" + async_logger.info(text) # CITATION: async_logger.info(text) + + def on_closing(self): # CITATION: def on_closing(self): + """Trigger an OS-level exit to cleanly bypass Tkinter background tasks.""" + self.destroy() # CITATION: self.destroy() + os._exit(0) # CITATION: os._exit(0) + + +class ReportsPage(ctk.CTkFrame): + """Page 2 Content Host. Organizes the Left Collapsible Panel and the Main Data Panel side-by-side.""" + + def __init__(self, master, controller, retries_var, backoff_var): + super().__init__(master, fg_color="transparent") + self.controller = controller + + # 1. Left Collapsible Navigation Sidebar + self.sidebar = SidebarFrame( + self, + disconnect_callback=self.controller.on_disconnect_clicked, + selection_callback=self.on_sidebar_selection_changed + ) + self.sidebar.pack(side="left", fill="y", padx=(0, 10)) + + # 2. Right-hand Main Dashboard Container + self.dashboard_container = ctk.CTkFrame(self, fg_color="transparent") + self.dashboard_container.pack(side="right", fill="both", expand=True) + + # Top Header Bar mimicking Google Workspace Deal Assistant details + self.nav_header = ctk.CTkFrame( + self.dashboard_container, + fg_color=COLOR_SURFACE, + height=70, + corner_radius=12, + border_width=1, + border_color=COLOR_OUTLINE_LIGHT + ) + self.nav_header.pack(fill="x", pady=(0, 15)) + self.nav_header.pack_propagate(False) + + # Text container frame to keep alignment clean next to the Action Button + self.header_text_frame = ctk.CTkFrame(self.nav_header, fg_color="transparent") + self.header_text_frame.pack(side="left", padx=20) + + self.nav_title = ctk.CTkLabel( + self.header_text_frame, + text="Usage Report", + font=ctk.CTkFont(family="Segoe UI", size=20, weight="bold"), + text_color=COLOR_TEXT_MAIN + ) + self.nav_title.pack(anchor="w") + + + + # 3. Fetch Report button on the right side of the header panel (Stage 2) + self.fetch_btn = ctk.CTkButton( + self.nav_header, + text="Fetch Report", + command=self.on_fetch_report_clicked, + width=150, + height=36, + corner_radius=8, + fg_color=COLOR_PRIMARY, + hover_color=COLOR_PRIMARY_HOVER, + font=FONT_BODY_BOLD + ) + self.fetch_btn.pack(side="right", padx=20, pady=17) + + # 4. Download PDF button next to Fetch Report + self.pdf_btn = ctk.CTkButton( + self.nav_header, + text="Download PDF", + command=self.on_download_pdf_clicked, + width=150, + height=36, + corner_radius=8, + fg_color="transparent", + border_width=1, + border_color=COLOR_PRIMARY, + text_color=COLOR_PRIMARY, + hover_color=COLOR_SECONDARY_HOVER, + font=FONT_BODY_BOLD, + state="disabled" + ) + self.pdf_btn.pack(side="right", padx=(0, 20), pady=17) + + + # Initialize the telemetry view (No TabView layout) + self.m365_telemetry_view = M365TelemetryTab( + master=self.dashboard_container, + log_callback=controller.log_msg, + retries_var=retries_var, + backoff_var=backoff_var + ) + self.m365_telemetry_view.pack(fill="both", expand=True) + self.m365_telemetry_view.on_all_done_callback = self.on_telemetry_fetch_completed + + # Initialize the migration planner view (initially hidden) + self.migration_planner_view = MigrationPlannerView( + master=self.dashboard_container, + controller=self.controller + ) + + # Initialize the user persona view (initially hidden) + self.user_persona_view = UserPersonaAnalysisView( + master=self.dashboard_container, + controller=self.controller + ) + + # Adapt layout recursively to hide original inputs from view + self.adapt_embedded_view() + + def on_sidebar_selection_changed(self, label): + import gc + if label == "Usage and adoption": + # Show Telemetry, Hide Migration Planner & Persona Analysis + self.migration_planner_view.pack_forget() + self.user_persona_view.pack_forget() + self.nav_title.configure(text="Usage Report") + self.fetch_btn.pack(side="right", padx=20, pady=17) + self.pdf_btn.pack(side="right", padx=(0, 20), pady=17) + self.m365_telemetry_view.pack(fill="both", expand=True) + gc.collect() + elif label == "Migration planner": + # Show Migration Planner, Hide Telemetry & Persona Analysis + self.m365_telemetry_view.pack_forget() + self.user_persona_view.pack_forget() + self.fetch_btn.pack_forget() + self.pdf_btn.pack_forget() + self.nav_title.configure(text="Migration Planner") + self.migration_planner_view.pack(fill="both", expand=True) + gc.collect() + elif "User Persona Analysis" in label: + # Show Persona Analysis, Hide Telemetry & Migration Planner + self.m365_telemetry_view.pack_forget() + self.migration_planner_view.pack_forget() + self.fetch_btn.pack_forget() + self.pdf_btn.pack_forget() + self.nav_title.configure(text="User Persona Analysis (Experimental)") + self.user_persona_view.pack(fill="both", expand=True) + gc.collect() + + def adapt_embedded_view(self): + """Traverses M365TelemetryTab to identify and hide native login components.""" + self.embedded_entries = [] + self.embedded_submit_btn = None + self.embedded_labels = [] + + def find_widgets_recursive(widget): + if isinstance(widget, ctk.CTkEntry): + self.embedded_entries.append(widget) + elif isinstance(widget, ctk.CTkButton): + btn_txt = str(widget.cget("text")).lower() + if "submit" in btn_txt or btn_txt == "": + self.embedded_submit_btn = widget + elif isinstance(widget, ctk.CTkLabel): + lbl_txt = str(widget.cget("text")).lower() + if any(kw in lbl_txt for kw in ["tenant id", "client id", "client secret", "connect your", "authenticate and audit"]): + self.embedded_labels.append(widget) + + for child in widget.winfo_children(): + find_widgets_recursive(child) + + find_widgets_recursive(self.m365_telemetry_view) + + # Remove the target widgets from layout grids/packs programmatically + for entry in self.embedded_entries: + entry.pack_forget() + entry.grid_forget() + + for label in self.embedded_labels: + label.pack_forget() + label.grid_forget() + + if self.embedded_submit_btn: + self.embedded_submit_btn.pack_forget() + self.embedded_submit_btn.grid_forget() + + if hasattr(self.m365_telemetry_view, "inputs_frame"): + self.m365_telemetry_view.inputs_frame.pack_forget() + self.m365_telemetry_view.inputs_frame.grid_forget() + + def on_fetch_report_clicked(self): + """Stage 2: Migrates stored variables into the telemetry coordinator and triggers fetch directly, or cancels if in progress.""" + if getattr(self.m365_telemetry_view, "is_fetching", False): + self.m365_telemetry_view.cancel_fetching() + return + + tenant = self.controller.stored_tenant + client = self.controller.stored_client + secret = self.controller.stored_secret + + if not tenant or not client or not secret: + logger.warning("Fetch Report triggered, but connection credentials are empty.") + return + + logger.info("Fetch Report triggered. Invoking background parallel audits...") + # Toggle button to Cancel and keep it enabled and active + self.fetch_btn.configure(state="normal", text="Cancel", fg_color="#DC2626") # Red color for cancel + self.pdf_btn.configure(state="disabled") + + # Set variables of m365_telemetry_view directly + self.m365_telemetry_view.lic_tenant_id.set(tenant) + self.m365_telemetry_view.lic_client_ids.set(client) + self.m365_telemetry_view.lic_client_secrets.set(secret) + + # Directly call the fetch command + self.m365_telemetry_view.authenticate_licenses_tab() + + def on_telemetry_fetch_completed(self, success: bool): + """Callback from M365TelemetryTab when all parallel reports complete or are cancelled.""" + self.fetch_btn.configure(state="normal", text="Fetch Report", fg_color=COLOR_PRIMARY) + + # Check if any data has been successfully fetched (enables partial downloads post-cancellation or partial failures) + data = self.m365_telemetry_view.get_all_telemetry_data() + directory = data.get("directory") or {} + has_any_data = any([ + data.get("skus"), + directory.get("domains"), + directory.get("group_counts"), + directory.get("user_counts"), + data.get("o365_usage"), + data.get("o365_trend"), + data.get("m365_apps"), + data.get("mailbox"), + data.get("calendar"), + data.get("sharepoint"), + data.get("onedrive"), + data.get("security_labels"), + data.get("retention_policies"), + data.get("power_automate") + ]) + + if has_any_data: + self.pdf_btn.configure(state="normal") + else: + self.pdf_btn.configure(state="disabled") + + def on_download_pdf_clicked(self): + """Prompts the user to save the M365 usage report as a detailed PDF file.""" + from tkinter import filedialog, messagebox + import datetime + from telemetry.pdf_report import generate_pdf_report + + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + f = filedialog.asksaveasfilename( + initialfile=f"m365_usage_report_{ts}.pdf", + defaultextension=".pdf", + filetypes=[("PDF Documents", "*.pdf"), ("All Files", "*.*")], + parent=self + ) + if not f: + return + + data = self.m365_telemetry_view.get_all_telemetry_data() + + try: + generate_pdf_report(data, f) + messagebox.showinfo("Export Successful", f"PDF report successfully saved to:\n{f}", parent=self) + except Exception as e: + logger.error("Failed to generate PDF report", exc_info=True) + messagebox.showerror("Export Failed", f"Failed to generate PDF report: {e}", parent=self) + + def clear_session_data(self): + """Wipes the cached parameters from telemetry objects and resets the Fetch button.""" + logger.info("Clearing session data in ReportsPage.") + for entry in self.embedded_entries: + entry.delete(0, "end") + + # Reset Fetch Report button state + self.fetch_btn.configure(state="normal", text="Fetch Report", fg_color="#1E3A8A") + self.pdf_btn.configure(state="disabled") + + # Reset the telemetry coordinator tab and hide all grids + self.m365_telemetry_view.reset_tab() + + # Reset migration planner view back to selector screen + self.migration_planner_view.show_selector() + + # Reset the sidebar selection state + self.sidebar.reset_selection() + # Switch back UI elements to default Usage report view + self.on_sidebar_selection_changed("Usage and adoption") + + +class SidebarFrame(ctk.CTkFrame): + """Collapsible Left Navigation Sidebar, matching Workspace Deal Assistant styling.""" + + def __init__(self, master, disconnect_callback, selection_callback, **kwargs): + # Increased initial width to 380px to avoid text truncation of longer menu items + super().__init__(master, width=380, fg_color=COLOR_SURFACE, corner_radius=12, border_width=1, border_color=COLOR_OUTLINE_LIGHT, **kwargs) + self.pack_propagate(False) # Lock sidebar panel dimensions + self.disconnect_callback = disconnect_callback + self.selection_callback = selection_callback + self.is_expanded = True + + # Header Branding Section + self.header_frame = ctk.CTkFrame(self, fg_color="transparent") + self.header_frame.pack(fill="x", padx=20, pady=(25, 30)) + + # Brand Icon representing a deal helper (Handshake 🤝) + self.logo_label = ctk.CTkLabel( + self.header_frame, + text="🤝", + font=ctk.CTkFont(family="Segoe UI", size=24), + text_color=COLOR_PRIMARY + ) + self.logo_label.pack(side="left", padx=(5, 5)) + + self.brand_text_area = ctk.CTkFrame(self.header_frame, fg_color="transparent") + self.brand_text_area.pack(side="left", fill="both", expand=True) + + self.brand_title = ctk.CTkLabel( + self.brand_text_area, + text="Deal Assistant", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN, + anchor="w" + ) + self.brand_title.pack(fill="x", pady=2) + + # Collapse / Expand control button + self.toggle_btn = ctk.CTkButton( + self, + text="⏴", + command=self.toggle_sidebar, + width=26, + height=28, + corner_radius=13, + fg_color=COLOR_SURFACE_VARIANT, + hover_color=COLOR_SECONDARY_HOVER, + text_color=COLOR_TEXT_SUB, + font=FONT_BODY_BOLD + ) + self.toggle_btn.place(relx=1.0, x=-22, y=26, anchor="center") + + # Vertical menu container (increased padding for larger sidebar) + self.menu_items_frame = ctk.CTkFrame(self, fg_color="transparent") + self.menu_items_frame.pack(fill="both", expand=True, padx=20) + + # Definitions for active/inactive routes + self.menu_buttons = [] + self.menu_data = [ + ("Usage and adoption", "📊", True), + ("Migration planner", "🚀", False), + ("User Persona Analysis (Experimental)", "👥", False) + ] + + self.render_navigation_menu() + + # Session closure button pinned safely at bottom + self.disconnect_row = ctk.CTkFrame(self, fg_color="transparent") + self.disconnect_row.pack(fill="x", side="bottom", padx=20, pady=25) + + self.disconnect_symbol = ctk.CTkLabel( + self.disconnect_row, + text="🚪", + font=ctk.CTkFont(family="Segoe UI", size=14), + text_color=COLOR_ERROR + ) + self.disconnect_symbol.pack(side="left", padx=(12, 6)) + + self.disconnect_btn = ctk.CTkButton( + self.disconnect_row, + text="Disconnect", + command=self.disconnect_callback, + anchor="w", + height=38, + corner_radius=8, + fg_color="transparent", + text_color=COLOR_ERROR, + hover_color="#FCE8E6", + font=FONT_BODY_MEDIUM + ) + self.disconnect_btn.pack(side="left", fill="x", expand=True) + + # RAM Usage Display Row (packed second with side="bottom", placing it directly above disconnect_row) + self.ram_row = ctk.CTkFrame(self, fg_color="transparent") + self.ram_row.pack(fill="x", side="bottom", padx=20, pady=(0, 10)) + + self.ram_symbol = ctk.CTkLabel( + self.ram_row, + text="💾", + font=ctk.CTkFont(family="Segoe UI", size=14), + text_color=COLOR_TEXT_SUB + ) + self.ram_symbol.pack(side="left", padx=(12, 6)) + + self.ram_lbl = ctk.CTkLabel( + self.ram_row, + text="RAM: Checking...", + anchor="w", + height=38, + text_color=COLOR_TEXT_SUB, + font=FONT_BODY_MEDIUM + ) + self.ram_lbl.pack(side="left", fill="x", expand=True) + + self._last_ram_log_time = 0 + # Start periodic RAM usage updates + self.update_ram_usage() + + def reset_selection(self): + self.menu_data = [ + ("Usage and adoption", "📊", True), + ("Migration planner", "🚀", False), + ("User Persona Analysis (Experimental)", "👥", False) + ] + self.render_navigation_menu() + + def on_menu_item_clicked(self, label): + for i, (m_label, m_icon, m_active) in enumerate(self.menu_data): + self.menu_data[i] = (m_label, m_icon, m_label == label) + self.render_navigation_menu() + self.selection_callback(label) + + + + def render_navigation_menu(self): + """Builds clean selection widgets mapping Workspace items.""" + for item in self.menu_buttons: + item[0].destroy() + item[1].destroy() + self.menu_buttons.clear() + + for label, icon, is_active in self.menu_data: + row_frame = ctk.CTkFrame(self.menu_items_frame, fg_color="transparent") + row_frame.pack(fill="x", pady=4) + + # Highlighting indicators mapping screenshot + if is_active: + btn_fg = COLOR_TONAL_BG + text_color = COLOR_PRIMARY + hover_color = "#D2E3FC" + weight = "bold" + else: + btn_fg = "transparent" + text_color = COLOR_TEXT_SUB + hover_color = COLOR_SURFACE_VARIANT + weight = "normal" + + icon_lbl = ctk.CTkLabel( + row_frame, + text=icon, + font=ctk.CTkFont(family="Segoe UI", size=15), + text_color=text_color + ) + icon_lbl.pack(side="left", padx=(12, 8)) + + btn = ctk.CTkButton( + row_frame, + text=label if self.is_expanded else "", + anchor="w", + height=38, + corner_radius=8, + fg_color=btn_fg, + text_color=text_color, + hover_color=hover_color, + state="normal", + font=ctk.CTkFont(family="Segoe UI", size=13, weight=weight), + command=lambda l=label: self.on_menu_item_clicked(l) + ) + btn.pack(side="left", fill="x", expand=True) + self.menu_buttons.append((row_frame, btn, icon_lbl)) + + + def toggle_sidebar(self): + """Performs layout adjustments to expand/collapse panel width dynamically.""" + if self.is_expanded: + # Shift width configuration to compact state (72px) + self.configure(width=72) + self.brand_text_area.pack_forget() + self.logo_label.pack(side="top", pady=10) + self.is_expanded = False + self.toggle_btn.configure(text="⏵") + + # Wipe text arrays inside panel items + for row, btn, icon in self.menu_buttons: + btn.configure(text="") + self.disconnect_btn.configure(text="") + self.ram_lbl.configure(text="") + logger.info("Sidebar collapsed.") + else: + # Return layout to expanded parameters (380px) + self.configure(width=380) + self.logo_label.pack(side="left", padx=(5, 5)) + self.brand_text_area.pack(side="left", fill="both", expand=True) + self.is_expanded = True + self.toggle_btn.configure(text="⏴") + + # Restore original strings dynamically + for idx, (row, btn, icon) in enumerate(self.menu_buttons): + btn.configure(text=self.menu_data[idx][0]) + self.disconnect_btn.configure(text="Disconnect") + self.update_ram_label_immediate() + logger.info("Sidebar expanded.") + + def update_ram_label_immediate(self): + """Updates the RAM label text immediately without waiting for the timer.""" + try: + import psutil + process = psutil.Process(os.getpid()) + ram_mb = process.memory_info().rss / (1024 * 1024) + if self.is_expanded: + self.ram_lbl.configure(text=f"RAM: {ram_mb:.1f} MB") + else: + self.ram_lbl.configure(text="") + + # Log RAM usage to log file every 10 seconds + import time + now = time.time() + if now - self._last_ram_log_time >= 10: + logger.info(f"App memory consumption: {ram_mb:.1f} MB") + self._last_ram_log_time = now + except Exception as e: + logger.error(f"Error checking RAM usage: {e}") + if self.is_expanded: + self.ram_lbl.configure(text="RAM: N/A") + else: + self.ram_lbl.configure(text="") + + + def update_ram_usage(self): + """Periodically updates the displayed RAM usage of the current process.""" + self.update_ram_label_immediate() + self._ram_timer_id = self.after(2000, self.update_ram_usage) + + + +def collect_power_automate_telemetry(tenant_id, client_id, client_secret, env_url): # CITATION: def collect_power_automate_telemetry(tenant_id, client_id, client_secret, env_url): + """Integrates the Power Automate scan into the telemetry execution flow.""" + logger.info("--- Power Automate Telemetry Phase Initiated ---") # CITATION: logger.info("--- Power Automate Telemetry Phase Initiated ---") + + if not env_url: # CITATION: if not env_url: + logger.warning("Skipping Power Automate: Environment URL not provided.") # CITATION: logger.warning("Skipping Power Automate: Environment URL not provided.") + return {} + + try: + scanner = PowerAutomateScanner(tenant_id, client_id, client_secret, env_url) # CITATION: scanner = PowerAutomateScanner(tenant_id, client_id, client_secret, env_url) + results = scanner.scan_flows() # CITATION: results = scanner.scan_flows() + + if results: # CITATION: if results: + logger.info(f"Telemetry Success: Aggregated data for {results['total_active_flows']} flows.") # CITATION: logger.info(f"Telemetry Success: Aggregated data for {results['total_active_flows']} flows.") + return results + else: + logger.error("Telemetry Warning: No flow data was returned from the scanner.") # CITATION: logger.error("Telemetry Warning: No flow data was returned from the scanner.") + return {} + + except Exception as e: # CITATION: except Exception as e: + logger.error(f"Critical Error during Power Automate scan: {str(e)}") # CITATION: logger.error(f"Critical Error during Power Automate scan: {str(e)}") + return {} + finally: + logger.info("--- Power Automate Telemetry Phase Concluded ---") # CITATION: logger.info("--- Power Automate Telemetry Phase Concluded ---") + + +if __name__ == "__main__": + ctk.set_appearance_mode("Light") # CITATION: ctk.set_appearance_mode("Light") + app = TelemetryApp() # CITATION: app = TelemetryApp() + app.mainloop() # CITATION: app.mainloop() diff --git a/flet_app/README.md b/flet_app/README.md new file mode 100644 index 00000000..73ea2366 --- /dev/null +++ b/flet_app/README.md @@ -0,0 +1,91 @@ +# Deal Assistant - Flet Dashboard UI + +This folder contains the Flet-based modern desktop dashboard application for the **Migration Planner Tool / Deal Assistant**. It provides an interactive, beautiful, web-like graphical user interface to collect and analyze Microsoft 365 tenant license adoption, compliance policies, active usage trends, and Power Automate flow telemetry. + +--- + +## Key Features + +- **Unified Credentials Connection (`AuthView`):** Enter your Tenant ID, Client ID, and Client Secret. +- **Certificate-Based Auth Flow (`CertInstructionsView`):** Automatically detects if a security certificate is missing. Generates a new `certificate.pem` file locally and guides you on uploading it to the Microsoft Entra ID portal to enable secure Exchange Online and Retention Policy scans. +- **Interactive Reports Dashboard (`DashboardView`):** + 1. **Subscribed SKUs Inventory Summary:** Lists active license plans, pre-paid quantities, consumed units, and status. Includes a **CSV Export** button. + 2. **O365 Active Users Usage:** Shows active user metrics (30-day, 90-day, 180-day) for Exchange, OneDrive, SharePoint, and Teams. + 3. **O365 30-Day Active User Trend:** Renders a gorgeous visual line chart (using Matplotlib backend rendering) showing historical trends. + 4. **M365 App Usage (180 Days):** Platform/App distribution details for user endpoints. + 5. **Exchange Online Mailbox Usage Telemetry:** Details total mailboxes, collective size, average sizes, and email volumes. + 6. **SharePoint Site Usage Telemetry:** Details total sites, storage consumed, files stored, and percent active files. + 7. **OneDrive Usage Telemetry:** Highlights OneDrive accounts, usage levels, file synchronisation percentages, and active OneNote users. + 8. **Sensitivity Labels:** Displays configured sensitivity labels (with child hierarchies), protection details, priority, and application targets. Supports **Pagination** (Page 1 of N). + 9. **Retention Compliance Policies:** Displays tenant compliance rules, workloads, and duration metrics. Features a quick link to open Microsoft Purview. + 10. **Power Automate Flows:** Lists environment counts, flow types, and premium/custom connector usage. Includes a **CSV Export** button to download complex logic flows. +- **Granular Individual Card Retry/Refresh:** Each card features a Refresh (`ft.Icons.REFRESH`) button on the top-right. You can re-fetch telemetry for an individual section (e.g. just SharePoint or just SKUs) without having to trigger a full master scan of the entire tenant again. + +--- + +## Prerequisites & Installation + +To run the Flet application, you need to set up Python and install the required UI and backend libraries. + +### 1. Python Environment +Make sure you have **Python 3.10** or newer installed. We highly recommend using a virtual environment: + +```bash +# Create a virtual environment +python -m venv venv + +# Activate the environment +# On macOS/Linux: +source venv/bin/activate +# On Windows (cmd): +.\venv\Scripts\activate +# On Windows (PowerShell): +.\venv\Scripts\Activate.ps1 +``` + +### 2. Install Required Python Packages +With your virtual environment active, run: + +```bash +pip install flet matplotlib pandas requests urllib3 aiohttp certifi psutil Pillow customtkinter +``` + +*Note: Flet does not require any additional web server setup. Matplotlib is used in headless mode (`matplotlib.use("Agg")`) to render the trend chart into the Flet UI natively.* + +### 3. PowerShell Prerequisites (For Retention Policy Scan) +The Retention Compliance Policy scanner uses PowerShell Core and the Exchange Online module. + +#### Install PowerShell Core (`pwsh`): +- **macOS (via Homebrew):** + ```bash + brew install powershell + ``` +- **Windows (via winget):** + ```cmd + winget install --id Microsoft.Powershell --source winget + ``` + +#### Install the Exchange Online Module: +Open PowerShell (`pwsh`) and install the module: +```powershell +Install-Module -Name ExchangeOnlineManagement -Scope CurrentUser -Force +``` + +--- + +## How to Run + +1. Open your terminal or command prompt and navigate to the project root directory (the parent of `flet_app/`): + ```bash + cd /path/to/project_root + ``` +2. Activate your virtual environment: + ```bash + source venv/bin/activate + ``` +3. Launch the Flet application: + ```bash + python flet_app/main.py + ``` +4. Enter your Azure Client/Tenant credentials to connect. +5. If requested, locate the auto-generated certificate (`certificate/certificate.pem`), upload it to the Azure portal under **App Registrations > Certificates & secrets**, and click **Continue** to load the dashboard. diff --git a/flet_app/auth_view.py b/flet_app/auth_view.py new file mode 100644 index 00000000..3507e9e8 --- /dev/null +++ b/flet_app/auth_view.py @@ -0,0 +1,86 @@ +import flet as ft +from flet_app.styles import * + +class AuthView(ft.Container): + def __init__(self, on_connect_clicked): + super().__init__() + self.on_connect_clicked = on_connect_clicked + + self.expand = True + + self.tenant_input = ft.TextField( + label="Tenant ID", + border_color=COLOR_OUTLINE, + focused_border_color=COLOR_PRIMARY, + text_size=14, + ) + self.client_input = ft.TextField( + label="Client ID", + border_color=COLOR_OUTLINE, + focused_border_color=COLOR_PRIMARY, + text_size=14, + ) + self.secret_input = ft.TextField( + label="Client Secret", + password=True, + can_reveal_password=True, + border_color=COLOR_OUTLINE, + focused_border_color=COLOR_PRIMARY, + text_size=14, + ) + + self.status_text = ft.Text(value="", color=COLOR_ERROR, size=14) + + self.content = ft.Column( + alignment=ft.MainAxisAlignment.CENTER, + horizontal_alignment=ft.CrossAxisAlignment.CENTER, + controls=[ + ft.Container( + width=600, + bgcolor=COLOR_SURFACE, + border_radius=12, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + padding=40, + content=ft.Column( + controls=[ + ft.Text("Deal Assistant", size=28, weight=ft.FontWeight.BOLD, color=COLOR_PRIMARY), + ft.Text("Connect your Azure App Credentials to begin auditing your tenant.", size=16, color=COLOR_TEXT_SUB), + ft.Divider(height=40, color="transparent"), + self.tenant_input, + ft.Divider(height=10, color="transparent"), + self.client_input, + ft.Divider(height=10, color="transparent"), + self.secret_input, + ft.Divider(height=10, color="transparent"), + self.status_text, + ft.Divider(height=20, color="transparent"), + ft.ElevatedButton( + content="Connect & Continue", + bgcolor=COLOR_PRIMARY, + color=ft.Colors.WHITE, + height=45, + style=ft.ButtonStyle(shape=ft.RoundedRectangleBorder(radius=8)), + on_click=self.handle_connect, + width=float('inf') + ) + ] + ) + ) + ] + ) + + def handle_connect(self, e): + tenant = self.tenant_input.value.strip() if self.tenant_input.value else "" + client = self.client_input.value.strip() if self.client_input.value else "" + secret = self.secret_input.value.strip() if self.secret_input.value else "" + + if not tenant or not client or not secret: + self.status_text.value = "Error: Tenant ID, Client ID, and Client Secret are required." + self.update() + return + + self.status_text.value = "" + self.update() + + # Pass the credentials to the parent callback + self.on_connect_clicked(tenant, client, secret) diff --git a/flet_app/cert_instructions_view.py b/flet_app/cert_instructions_view.py new file mode 100644 index 00000000..2308dd43 --- /dev/null +++ b/flet_app/cert_instructions_view.py @@ -0,0 +1,75 @@ +import flet as ft +from flet_app.styles import * + +class CertInstructionsView(ft.Container): + def __init__(self, pem_path, client_id, on_continue): + super().__init__() + self.pem_path = pem_path + self.client_id = client_id + self.on_continue = on_continue + + self.expand = True + + self.content = ft.Column( + alignment=ft.MainAxisAlignment.CENTER, + horizontal_alignment=ft.CrossAxisAlignment.STRETCH, + controls=[ + ft.Container( + bgcolor=COLOR_SURFACE, + border_radius=12, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + padding=40, + margin=ft.margin.symmetric(horizontal=40), + content=ft.Column( + controls=[ + ft.Text("Certificate Upload", size=24, weight=ft.FontWeight.BOLD, color=COLOR_PRIMARY), + ft.Text("A new security certificate has been generated for hybrid authentication.\n\nUploading this certificate is highly recommended, but optional:", size=14, color=COLOR_TEXT_MAIN), + ft.Text("• If you UPLOAD the certificate:\n All report sections will be fully functional.", size=14, color=COLOR_SUCCESS, weight=ft.FontWeight.BOLD), + ft.Text("• If you SKIP uploading the certificate:\n You can still run the reports. However, sections relying on certificate-based authentication (such as detailed Calendar settings, Shared/Public mailbox statistics, Retention Policies etc.) will be skipped and show as unavailable.", size=14, color=COLOR_ERROR, weight=ft.FontWeight.BOLD), + ft.Divider(height=20, color="transparent"), + + ft.Container( + content=ft.Column([ + ft.Text("Upload Instructions", size=14, weight=ft.FontWeight.BOLD, color=COLOR_TONAL_TEXT), + ft.Text("1. Locate the certificate file generated at:", size=13, color=COLOR_TEXT_MAIN), + ft.Container( + content=ft.Text(self.pem_path, size=12, color=COLOR_TEXT_MAIN, selectable=True, font_family="Courier New"), + bgcolor=COLOR_SURFACE, + padding=10, + border_radius=6, + width=float('inf') + ), + ft.Text("2. Log in to the Microsoft Azure portal and navigate to the App Registration with Client ID:", size=13, color=COLOR_TEXT_MAIN), + ft.Container( + content=ft.Text(self.client_id, size=12, color=COLOR_TEXT_MAIN, selectable=True, font_family="Courier New"), + bgcolor=COLOR_SURFACE, + padding=10, + border_radius=6, + width=float('inf') + ), + ft.Text("3. Upload the certificate under:", size=13, color=COLOR_TEXT_MAIN), + ft.Text(" Certificates & secrets -> Certificates -> Upload certificate", size=13, weight=ft.FontWeight.BOLD, color=COLOR_TEXT_MAIN), + ]), + border=ft.Border.all(1, COLOR_OUTLINE), + border_radius=8, + padding=20, + bgcolor=COLOR_TONAL_BG + ), + + ft.Divider(height=30, color="transparent"), + ft.ElevatedButton( + content=ft.Text("Continue", color=ft.Colors.WHITE, weight=ft.FontWeight.BOLD), + bgcolor=COLOR_PRIMARY, + height=40, + style=ft.ButtonStyle(shape=ft.RoundedRectangleBorder(radius=20)), + on_click=self.handle_continue, + width=float('inf') + ) + ] + ) + ) + ] + ) + + def handle_continue(self, e): + self.on_continue() diff --git a/flet_app/custom_chart.py b/flet_app/custom_chart.py new file mode 100644 index 00000000..8133c2ca --- /dev/null +++ b/flet_app/custom_chart.py @@ -0,0 +1,59 @@ +import flet as ft +import io +import base64 +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt + +class CustomLineChart(ft.Container): + def __init__(self, dates, datasets, colors, height=300): + super().__init__() + self.dates = dates + self.datasets = datasets + self.colors = colors + self.chart_height = height + + self.content = self.create_chart_image() + self.height = self.chart_height + self.expand = True + + def create_chart_image(self): + if not self.dates or not self.datasets: + return ft.Container() + + fig, ax = plt.subplots(figsize=(10, 4)) + + for name, data in self.datasets.items(): + color = self.colors.get(name, "#000000") + ax.plot(self.dates, data, label=name, color=color, marker='o', markersize=5, linewidth=2.5) + + # Formatting + fig.patch.set_facecolor('none') + ax.set_facecolor('none') + ax.spines['top'].set_visible(False) + ax.spines['right'].set_visible(False) + ax.spines['left'].set_color('#E2E8F0') + ax.spines['bottom'].set_color('#E2E8F0') + ax.grid(axis='y', color='#E2E8F0', linestyle='-', linewidth=1) + ax.tick_params(axis='both', colors='#64748B', labelsize=10) + + # Adjust x ticks to not overlap + num_dates = len(self.dates) + label_interval = max(1, num_dates // 6) + ax.set_xticks(range(0, num_dates, label_interval)) + ax.set_xticklabels([self.dates[i] for i in range(0, num_dates, label_interval)]) + + # Apply tight layout to minimize whitespace + fig.tight_layout() + + # Save to buffer + buf = io.BytesIO() + fig.savefig(buf, format='png', bbox_inches='tight', transparent=True, dpi=120) + buf.seek(0) + + # Close fig + plt.close(fig) + + b64_string = base64.b64encode(buf.read()).decode('utf-8') + + return ft.Image(src=f"data:image/png;base64,{b64_string}", fit="contain", expand=True) diff --git a/flet_app/dashboard.py b/flet_app/dashboard.py new file mode 100644 index 00000000..f36c4b15 --- /dev/null +++ b/flet_app/dashboard.py @@ -0,0 +1,1184 @@ +import flet as ft +from flet_app.styles import * +from flet_app.sidebar import Sidebar +from flet_app.custom_chart import CustomLineChart +import threading +import sys +import os +import datetime +import csv + +# Import existing backend modules +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from core.graph.client import GraphClient +from core.graph.directory import DirectoryService +from telemetry import active_users_usage as usage +from telemetry.power_automate import PowerAutomateScanner +from telemetry.mailbox_usage import run_mailbox_usage_pipeline +from telemetry.calendar_telemetry import run_calendar_telemetry_pipeline +from telemetry.sharepoint_onedrive_usage import run_sharepoint_pipeline, run_onedrive_pipeline +from telemetry.data_security_governance import run_security_governance_pipeline + +class DashboardView(ft.Container): + def __init__(self, tenant, client, secret, on_disconnect): + super().__init__() + self.tenant = tenant + self.client = client + self.secret = secret + self.on_disconnect = on_disconnect + + self.expand = True + + self.sidebar = Sidebar(on_disconnect=self.on_disconnect) + + # Saved data for CSV exports + self.last_licenses_items = [] + self.last_complex_flows = [] + + # Saved data for Sensitivity Labels pagination + self.flattened_labels = [] + self.current_labels_page = 0 + self.labels_per_page = 8 + + # Track states of parallel fetches + self.fetch_statuses = {} + + # Header + self.fetch_btn = ft.ElevatedButton( + content=ft.Text("Fetch Report", weight=ft.FontWeight.BOLD), + bgcolor=COLOR_PRIMARY, + color=ft.Colors.WHITE, + height=40, + style=ft.ButtonStyle(shape=ft.RoundedRectangleBorder(radius=8)), + on_click=self.handle_fetch + ) + + self.header = ft.Container( + content=ft.Row( + controls=[ + ft.Text("Usage Report", size=20, weight=ft.FontWeight.BOLD, color=COLOR_TEXT_MAIN), + self.fetch_btn + ], + alignment=ft.MainAxisAlignment.SPACE_BETWEEN + ), + bgcolor=COLOR_SURFACE, + border_radius=12, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + padding=ft.Padding.symmetric(horizontal=20, vertical=15), + margin=ft.Margin.only(bottom=20) + ) + + # 1. SKUs Card (with Export Button) + self.export_sku_btn = ft.IconButton( + icon=ft.Icons.DOWNLOAD, + icon_color=COLOR_PRIMARY, + tooltip="Export Spreadsheet", + disabled=True, + on_click=self.handle_export_skus + ) + self.sku_link = ft.TextButton( + content=ft.Text("Service Plan Reference ↗", color=COLOR_PRIMARY, weight=ft.FontWeight.BOLD), + on_click=lambda e: e.page.launch_url("https://learn.microsoft.com/en-us/entra/identity/users/licensing-service-plan-reference") + ) + sku_actions = ft.Row( + controls=[ + self.sku_link, + self.export_sku_btn + ], + spacing=5 + ) + self.sku_section = self.create_card("Subscribed SKUs", action_control=sku_actions, on_retry=self.handle_retry_skus) + + # 2. O365 Usage Card + self.o365_section = self.create_card("O365 Active Users Usage", on_retry=self.handle_retry_o365) + + # 3. O365 Trend Chart Card + self.trend_section = self.create_card("O365 30-Day Active User Trend", on_retry=self.handle_retry_trend) + + # 4. M365 App Usage Card + self.m365_section = self.create_card("M365 App Usage (180 Days)", on_retry=self.handle_retry_m365) + + # 5. Exchange Online Card (Combined Emails and Calendar Sections) + self.emails_container = ft.Container(content=ft.Text("No data yet.", color=COLOR_TEXT_SUB)) + self.calendar_container = ft.Container(content=ft.Text("No data yet.", color=COLOR_TEXT_SUB)) + self.exchange_layout = ft.Column( + controls=[ + ft.Text("Exchange Online Email", size=16, weight=ft.FontWeight.BOLD, color=COLOR_PRIMARY), + self.emails_container, + ft.Divider(height=10, color="transparent"), + ft.Text("Exchange Online Calendar", size=16, weight=ft.FontWeight.BOLD, color=COLOR_PRIMARY), + self.calendar_container + ], + spacing=5 + ) + self.exchange_section = self.create_card("Exchange Online", on_retry=self.handle_retry_exchange) + self.exchange_section.content_container.content = self.exchange_layout + + # 6. Files Card (Combined SharePoint and OneDrive Sections) + self.sharepoint_container = ft.Container(content=ft.Text("No data yet.", color=COLOR_TEXT_SUB)) + self.onedrive_container = ft.Container(content=ft.Text("No data yet.", color=COLOR_TEXT_SUB)) + self.files_layout = ft.Column( + controls=[ + ft.Text("SharePoint Site Usage (180 Days)", size=16, weight=ft.FontWeight.BOLD, color=COLOR_PRIMARY), + self.sharepoint_container, + ft.Divider(height=10, color="transparent"), + ft.Text("OneDrive Usage (180 Days)", size=16, weight=ft.FontWeight.BOLD, color=COLOR_PRIMARY), + self.onedrive_container + ], + spacing=5 + ) + self.files_section = self.create_card("Files", on_retry=self.handle_retry_files) + self.files_section.content_container.content = self.files_layout + + # 8. Sensitivity Labels Card (with Pagination Controls) + self.labels_pagination_info = ft.Text("Page 1 of 1", size=13, color=COLOR_TEXT_MAIN) + self.labels_prev_btn = ft.IconButton(ft.Icons.ARROW_BACK, on_click=self.handle_labels_prev, disabled=True) + self.labels_next_btn = ft.IconButton(ft.Icons.ARROW_FORWARD, on_click=self.handle_labels_next, disabled=True) + self.labels_pagination_row = ft.Row( + controls=[self.labels_prev_btn, self.labels_pagination_info, self.labels_next_btn], + alignment=ft.MainAxisAlignment.CENTER, + visible=False + ) + self.purview_labels_btn = ft.TextButton( + content=ft.Text("Open Purview Sensitivity Label Portal ↗", color=COLOR_PRIMARY, weight=ft.FontWeight.BOLD), + on_click=lambda e: e.page.launch_url("https://purview.microsoft.com/informationprotection/informationprotectionlabels/sensitivitylabels") + ) + self.labels_section = self.create_card( + "Sensitivity Labels", + action_control=self.purview_labels_btn, + bottom_control=self.labels_pagination_row, + on_retry=self.handle_retry_labels + ) + + # 9. Retention Policies Card (with Purview Link) + self.purview_btn = ft.TextButton( + content=ft.Text("Open Purview Retention Policy Portal ↗", color=COLOR_PRIMARY, weight=ft.FontWeight.BOLD), + on_click=lambda e: e.page.launch_url("https://purview.microsoft.com/datalifecyclemanagement/retention") + ) + self.retention_section = self.create_card("Retention Compliance Policies", action_control=self.purview_btn, on_retry=self.handle_retry_retention) + + # 9c. eDiscovery Cases Card (Instructional) + self.ediscovery_portal_btn = ft.TextButton( + content=ft.Text("Open Purview eDiscovery Portal ↗", color=COLOR_PRIMARY, weight=ft.FontWeight.BOLD), + on_click=lambda e: e.page.launch_url("https://purview.microsoft.com/ediscovery/casespage") + ) + self.ediscovery_permissions_btn = ft.TextButton( + content=ft.Text("Open Purview Permissions Settings ↗", color=COLOR_PRIMARY, weight=ft.FontWeight.BOLD), + on_click=lambda e: e.page.launch_url("https://purview.microsoft.com/settings/purviewpermissions") + ) + + self.ediscovery_section = self.create_card( + "eDiscovery Cases", + action_control=self.ediscovery_portal_btn + ) + + self.ediscovery_section.content_container.content = ft.Column([ + ft.Text( + "eDiscovery cases cannot be scanned directly under standard Application permissions. " + "To view your active cases, please navigate to Microsoft Purview on behalf of a user who has the eDiscovery Manager role.", + color=COLOR_TEXT_MAIN, + size=14 + ), + ft.Divider(height=10, color="transparent"), + ft.Row([ + ft.Text("To assign the eDiscovery Manager role, go to:", color=COLOR_TEXT_SUB, size=13), + self.ediscovery_permissions_btn + ], alignment=ft.MainAxisAlignment.START, spacing=5) + ], spacing=10) + + # 10. Power Automate Card (with Export Button) + self.export_pa_btn = ft.IconButton( + icon=ft.Icons.DOWNLOAD, + icon_color=COLOR_PRIMARY, + tooltip="Export Complex Flows", + disabled=True, + on_click=self.handle_export_pa + ) + self.pa_section = self.create_card("Power Automate", action_control=self.export_pa_btn, on_retry=self.handle_retry_pa) + + self.content_area = ft.Column( + controls=[ + self.sku_section, + self.o365_section, + self.trend_section, + self.m365_section, + self.exchange_section, + self.files_section, + self.labels_section, + self.retention_section, + self.ediscovery_section, + self.pa_section + ], + scroll=ft.ScrollMode.AUTO, + expand=True, + spacing=20 + ) + + self.content = ft.Row( + controls=[ + self.sidebar, + ft.Container( + content=ft.Column( + controls=[self.header, self.content_area], + expand=True + ), + expand=True, + padding=ft.Padding.only(left=20) + ) + ], + expand=True + ) + + def create_card(self, title, action_control=None, bottom_control=None, on_retry=None): + content_container = ft.Container(content=ft.Text("No data yet.", color=COLOR_TEXT_SUB)) + header_controls = [ft.Text(title, size=16, weight=ft.FontWeight.BOLD, color=COLOR_TEXT_MAIN)] + + actions = [] + if action_control: + actions.append(action_control) + + retry_btn = None + if on_retry: + retry_btn = ft.IconButton( + icon=ft.Icons.REFRESH, + icon_color=COLOR_PRIMARY, + icon_size=20, + tooltip="Refresh this section", + on_click=on_retry + ) + actions.append(retry_btn) + + if actions: + header_controls.append(ft.Row(controls=actions, spacing=10)) + + header_row = ft.Row( + controls=header_controls, + alignment=ft.MainAxisAlignment.SPACE_BETWEEN + ) + + column_controls = [ + header_row, + ft.Divider(height=20, color="transparent"), + content_container + ] + if bottom_control: + column_controls.append(bottom_control) + + card = ft.Container( + content=ft.Column(column_controls), + bgcolor=COLOR_SURFACE, + border_radius=12, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + padding=20, + width=float('inf') + ) + card.content_container = content_container + card.retry_btn = retry_btn + return card + + def set_loading(self, card, message): + card.content_container.content = ft.Column([ + ft.ProgressRing(), + ft.Text(message, color=COLOR_TEXT_SUB) + ], alignment=ft.MainAxisAlignment.CENTER, horizontal_alignment=ft.CrossAxisAlignment.CENTER) + if hasattr(card, "retry_btn") and card.retry_btn: + card.retry_btn.disabled = True + try: + card.retry_btn.update() + except Exception: + pass + + def set_tab_loading(self, tab_container, message): + tab_container.content = ft.Column([ + ft.ProgressRing(width=30, height=30), + ft.Text(message, color=COLOR_TEXT_SUB, size=13) + ], alignment=ft.MainAxisAlignment.CENTER, horizontal_alignment=ft.CrossAxisAlignment.CENTER) + + def set_error(self, card, message): + card.content_container.content = ft.Text(f"Error: {message}", color=COLOR_ERROR) + if hasattr(card, "retry_btn") and card.retry_btn: + card.retry_btn.disabled = False + try: + card.retry_btn.update() + except Exception: + pass + + def clear_loading(self, card): + if hasattr(card, "retry_btn") and card.retry_btn: + card.retry_btn.disabled = False + try: + card.retry_btn.update() + except Exception: + pass + + def start_individual_fetch(self, key, section, message, target): + self.fetch_btn.disabled = True + self.fetch_btn.update() + self.fetch_statuses[key] = "pending" + self.set_loading(section, message) + section.update() + threading.Thread(target=target, daemon=True).start() + + def handle_retry_skus(self, e): + self.start_individual_fetch("sku", self.sku_section, "Fetching SKU inventories...", self.fetch_skus) + + def handle_retry_o365(self, e): + self.start_individual_fetch("o365", self.o365_section, "Downloading O365 Active User reports...", self.fetch_o365) + + def handle_retry_trend(self, e): + self.start_individual_fetch("trend", self.trend_section, "Downloading O365 Trend report...", self.fetch_trend) + + def handle_retry_m365(self, e): + self.start_individual_fetch("m365", self.m365_section, "Downloading M365 App reports...", self.fetch_m365) + + def handle_retry_exchange(self, e): + self.fetch_btn.disabled = True + self.fetch_btn.update() + self.fetch_statuses["mailbox"] = "pending" + self.fetch_statuses["calendar"] = "pending" + self.set_tab_loading(self.emails_container, "Downloading Mailbox reports...") + self.set_tab_loading(self.calendar_container, "Querying Calendar settings...") + self.exchange_section.update() + threading.Thread(target=self.fetch_mailbox, daemon=True).start() + threading.Thread(target=self.fetch_calendar, daemon=True).start() + + def handle_retry_files(self, e): + self.fetch_btn.disabled = True + self.fetch_btn.update() + self.fetch_statuses["sharepoint"] = "pending" + self.fetch_statuses["onedrive"] = "pending" + self.set_tab_loading(self.sharepoint_container, "Downloading SharePoint reports...") + self.set_tab_loading(self.onedrive_container, "Downloading OneDrive reports...") + self.files_section.update() + threading.Thread(target=self.fetch_sharepoint, daemon=True).start() + threading.Thread(target=self.fetch_onedrive, daemon=True).start() + + def handle_retry_labels(self, e): + self.start_individual_fetch("labels", self.labels_section, "Retrieving Sensitivity labels...", self.fetch_labels) + + def handle_retry_retention(self, e): + self.start_individual_fetch("retention", self.retention_section, "Retrieving Retention policies...", self.fetch_retention) + + def handle_retry_pa(self, e): + self.start_individual_fetch("pa", self.pa_section, "Scanning Power Automate flows...", self.fetch_pa) + + def handle_fetch(self, e): + self.fetch_btn.disabled = True + self.fetch_btn.content = ft.Text("Fetching...", color=ft.Colors.WHITE) + self.fetch_btn.update() + + # Initialize fetch statuses + self.fetch_statuses = { + "sku": "pending", + "o365": "pending", + "trend": "pending", + "m365": "pending", + "mailbox": "pending", + "calendar": "pending", + "sharepoint": "pending", + "onedrive": "pending", + "labels": "pending", + "retention": "pending", + "pa": "pending" + } + + # 1. SKUs + self.set_loading(self.sku_section, "Fetching SKU inventories...") + self.sku_section.update() + threading.Thread(target=self.fetch_skus, daemon=True).start() + + # 2. O365 Usage + self.set_loading(self.o365_section, "Downloading O365 Active User reports...") + self.o365_section.update() + threading.Thread(target=self.fetch_o365, daemon=True).start() + + # 3. O365 Trend + self.set_loading(self.trend_section, "Downloading O365 Trend report...") + self.trend_section.update() + threading.Thread(target=self.fetch_trend, daemon=True).start() + + # 4. M365 App Usage + self.set_loading(self.m365_section, "Downloading M365 App reports...") + self.m365_section.update() + threading.Thread(target=self.fetch_m365, daemon=True).start() + + # 5. Exchange Online (Mailbox & Calendar) + self.set_tab_loading(self.emails_container, "Downloading Mailbox reports...") + self.set_tab_loading(self.calendar_container, "Querying Calendar settings...") + self.exchange_section.update() + threading.Thread(target=self.fetch_mailbox, daemon=True).start() + threading.Thread(target=self.fetch_calendar, daemon=True).start() + + # 6. Files (SharePoint & OneDrive) + self.set_tab_loading(self.sharepoint_container, "Downloading SharePoint reports...") + self.set_tab_loading(self.onedrive_container, "Downloading OneDrive reports...") + self.files_section.update() + threading.Thread(target=self.fetch_sharepoint, daemon=True).start() + threading.Thread(target=self.fetch_onedrive, daemon=True).start() + + # 8 & 9. Sensitivity Labels & Retention Policies + self.set_loading(self.labels_section, "Retrieving Sensitivity labels...") + self.labels_section.update() + threading.Thread(target=self.fetch_labels, daemon=True).start() + + self.set_loading(self.retention_section, "Retrieving Retention policies...") + self.retention_section.update() + threading.Thread(target=self.fetch_retention, daemon=True).start() + + # 10. Power Automate + self.set_loading(self.pa_section, "Scanning Power Automate flows...") + self.pa_section.update() + threading.Thread(target=self.fetch_pa, daemon=True).start() + + def mark_complete(self, key, status): + self.fetch_statuses[key] = status + if key in ["mailbox", "calendar"]: + if self.fetch_statuses.get("mailbox") != "pending" and self.fetch_statuses.get("calendar") != "pending": + self.clear_loading(self.exchange_section) + if self.page: + self.exchange_section.update() + elif key in ["sharepoint", "onedrive"]: + if self.fetch_statuses.get("sharepoint") != "pending" and self.fetch_statuses.get("onedrive") != "pending": + self.clear_loading(self.files_section) + if self.page: + self.files_section.update() + self.check_all_done() + + def check_all_done(self): + if not self.fetch_statuses: + return + if "pending" not in self.fetch_statuses.values(): + self.fetch_btn.disabled = False + self.fetch_btn.content = ft.Text("Fetch Report", weight=ft.FontWeight.BOLD) + if self.page: + self.fetch_btn.update() + + # --- Fetching Logic --- + + def fetch_skus(self): + try: + client = GraphClient(tenant_id=self.tenant, client_ids=self.client, client_secrets=self.secret, concurrency=1, retries=30, backoff=2) + client.authenticate(required_scopes=["Organization.Read.All", "Directory.Read.All"]) + dir_service = DirectoryService(client) + sku_data = dir_service.get_subscribed_skus() + client.close() + + items = sku_data.get("value", []) + self.last_licenses_items = items + + if not items: + self.sku_section.content_container.content = ft.Text("No subscribed product configurations found.", color=COLOR_TEXT_SUB) + self.export_sku_btn.disabled = True + else: + items.sort(key=lambda x: len(x.get("servicePlans", [])), reverse=True) + rows = [] + for item in items: + prepaid = item.get("prepaidUnits", {}) + p_str = f"Enabled: {prepaid.get('enabled', 0):,}" + if prepaid.get('warning', 0) > 0: p_str += f"\nWarn: {prepaid.get('warning'):,}" + if prepaid.get('suspended', 0) > 0: p_str += f"\nSusp: {prepaid.get('suspended'):,}" + rows.append(ft.DataRow(cells=[ + ft.DataCell(ft.Text(item.get("skuPartNumber", "UNKNOWN_SKU"), weight=ft.FontWeight.BOLD)), + ft.DataCell(ft.Text(p_str)), + ft.DataCell(ft.Text(f"{item.get('consumedUnits', 0):,}")) + ])) + + table = ft.DataTable( + columns=[ + ft.DataColumn(ft.Text("SKU Part Number", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Units", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Consumed Units", weight=ft.FontWeight.BOLD)), + ], + rows=rows, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + border_radius=8, + heading_row_color=COLOR_TONAL_BG, + ) + self.sku_section.content_container.content = ft.Column([table], scroll=ft.ScrollMode.AUTO, height=300) + self.export_sku_btn.disabled = False + + self.mark_complete("sku", "success") + except Exception as e: + self.set_error(self.sku_section, str(e)) + self.export_sku_btn.disabled = True + self.mark_complete("sku", "error") + finally: + self.clear_loading(self.sku_section) + if self.page: + self.sku_section.update() + self.export_sku_btn.update() + + def fetch_o365(self): + try: + try: + o365_data = usage.run_o365_pipeline(self.client, self.secret, self.tenant) + except Exception as o365_err: + script_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + local_file = os.path.join(script_dir, "telemetry", "reports", f"{self.tenant}_{self.client}", "Office365ActiveUserDetail(180d).csv") + if not os.path.exists(local_file): + local_file = os.path.join(script_dir, "telemetry", "reports", "Office365ActiveUserDetail(180d).csv") + if os.path.exists(local_file): + print(f"Falling back to local O365 file: {local_file}") + o365_data = usage.process_active_user_detail(local_file) + else: + raise o365_err + if not o365_data: + self.o365_section.content_container.content = ft.Text("No O365 usage data found.", color=COLOR_TEXT_SUB) + else: + rows = [] + for row_data in o365_data: + rows.append(ft.DataRow(cells=[ + ft.DataCell(ft.Text(str(row_data[0]), weight=ft.FontWeight.BOLD)), + ft.DataCell(ft.Text(f"{row_data[1]:,}")), + ft.DataCell(ft.Text(f"{row_data[2]:,}")), + ft.DataCell(ft.Text(f"{row_data[3]:,}")) + ])) + + table = ft.DataTable( + columns=[ + ft.DataColumn(ft.Text("Service", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("30 Days", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("90 Days", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("180 Days", weight=ft.FontWeight.BOLD)), + ], + rows=rows, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + border_radius=8, + heading_row_color=COLOR_TONAL_BG, + ) + self.o365_section.content_container.content = table + + self.mark_complete("o365", "success") + except Exception as e: + self.set_error(self.o365_section, str(e)) + self.mark_complete("o365", "error") + finally: + self.clear_loading(self.o365_section) + if self.page: + self.o365_section.update() + + def fetch_trend(self): + try: + try: + trend_data = usage.run_o365_trend_pipeline(self.client, self.secret, self.tenant) + except Exception as trend_err: + script_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + local_file = os.path.join(script_dir, "telemetry", "reports", f"{self.tenant}_{self.client}", "Office365ActiveUserCounts(30d).csv") + if not os.path.exists(local_file): + local_file = os.path.join(script_dir, "telemetry", "reports", "Office365ActiveUserCounts(30d).csv") + if os.path.exists(local_file): + print(f"Falling back to local trend file: {local_file}") + trend_data = usage.process_active_user_counts(local_file) + else: + raise trend_err + if not trend_data or not trend_data.get("dates"): + self.trend_section.content_container.content = ft.Text("No O365 trend data found.", color=COLOR_TEXT_SUB) + else: + dates = trend_data["dates"] + datasets = { + "Office 365": trend_data["office365"], + "Exchange": trend_data["exchange"], + "OneDrive": trend_data["onedrive"], + "SharePoint": trend_data["sharepoint"], + "Teams": trend_data["teams"] + } + colors = { + "Office 365": COLOR_PRIMARY, + "Exchange": "#C2410C", + "OneDrive": "#3B82F6", + "SharePoint": "#15803D", + "Teams": "#9333EA" + } + + chart = CustomLineChart(dates=dates, datasets=datasets, colors=colors, height=300) + + # Legend container + def legend_item(label, color): + return ft.Row([ + ft.Container(width=12, height=12, bgcolor=color, border_radius=3), + ft.Text(label, size=12, weight=ft.FontWeight.W_500, color=COLOR_TEXT_MAIN) + ], spacing=5) + + legend = ft.Row([ + legend_item("Office 365", colors["Office 365"]), + legend_item("Exchange", colors["Exchange"]), + legend_item("OneDrive", colors["OneDrive"]), + legend_item("SharePoint", colors["SharePoint"]), + legend_item("Teams", colors["Teams"]), + ], alignment=ft.MainAxisAlignment.CENTER, spacing=20) + + self.trend_section.content_container.content = ft.Column([ + chart, + ft.Divider(height=10, color="transparent"), + legend + ]) + + self.mark_complete("trend", "success") + except Exception as e: + self.set_error(self.trend_section, str(e)) + self.mark_complete("trend", "error") + finally: + self.clear_loading(self.trend_section) + if self.page: + self.trend_section.update() + + def fetch_m365(self): + try: + try: + m365_data = usage.run_m365_pipeline(self.client, self.secret, self.tenant) + except Exception as m365_err: + script_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + local_file = os.path.join(script_dir, "telemetry", "reports", f"{self.tenant}_{self.client}", "M365AppUserDetail(180d).csv") + if not os.path.exists(local_file): + local_file = os.path.join(script_dir, "telemetry", "reports", "M365AppUserDetail(180d).csv") + if os.path.exists(local_file): + print(f"Falling back to local M365 file: {local_file}") + m365_data = usage.process_m365_app_user_detail(local_file) + else: + raise m365_err + if not m365_data: + self.m365_section.content_container.content = ft.Text("No M365 App usage data found.", color=COLOR_TEXT_SUB) + else: + rows = [] + for row in m365_data: + rows.append(ft.DataRow(cells=[ + ft.DataCell(ft.Text(str(row[0]), weight=ft.FontWeight.BOLD)), + ft.DataCell(ft.Text(f"{row[1]:,}")) + ])) + + table = ft.DataTable( + columns=[ + ft.DataColumn(ft.Text("App / Platform", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Users Count", weight=ft.FontWeight.BOLD)), + ], + rows=rows, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + border_radius=8, + heading_row_color=COLOR_TONAL_BG, + ) + self.m365_section.content_container.content = ft.Column([table], scroll=ft.ScrollMode.AUTO, height=300) + + self.mark_complete("m365", "success") + except Exception as e: + self.set_error(self.m365_section, str(e)) + self.mark_complete("m365", "error") + finally: + self.clear_loading(self.m365_section) + if self.page: + self.m365_section.update() + + def fetch_mailbox(self): + try: + data = run_mailbox_usage_pipeline(self.client, self.secret, self.tenant) + rows_data = [ + ("Total Mailboxes Analyzed", f"{data.get('total_mailboxes', 0):,} Mailboxes"), + ("Total Size of All Mailboxes", data.get("total_storage_formatted", "0.00 Bytes")), + ("Average Mailbox Size", data.get("average_mailbox_size_formatted", "0.00 Bytes")), + ("Total Number of Emails", f"{data.get('total_emails', 0):,} Emails"), + ("Average Emails per User", f"{data.get('average_emails', 0.0):,.0f} Emails") + ] + if data.get("has_powershell"): + rows_data += [ + ("Shared Mailboxes Count", f"{data.get('shared_mailboxes_count', 0):,} Shared Mailboxes"), + ("Total Shared Mailbox Size", data.get('shared_mailboxes_total_formatted', "0.00 Bytes")), + ("Public Folders Count", f"{data.get('public_folders_count', 0):,} Public Folders"), + ("Total Public Folder Size", data.get('public_folders_total_formatted', "0.00 Bytes")) + ] + + rows = [ft.DataRow(cells=[ + ft.DataCell(ft.Text(str(r[0]), weight=ft.FontWeight.BOLD)), + ft.DataCell(ft.Text(str(r[1]))) + ]) for r in rows_data] + + table = ft.DataTable( + columns=[ + ft.DataColumn(ft.Text("Mailbox Metric Description", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Value / Measurement", weight=ft.FontWeight.BOLD)), + ], + rows=rows, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + border_radius=8, + heading_row_color=COLOR_TONAL_BG, + ) + + # Check warnings + if data.get("powershell_error"): + self.emails_container.content = ft.Column([ + ft.Text(f"⚠️ Warning: PowerShell stats failed. {data['powershell_error']}", color=COLOR_WARNING, size=13), + table + ], scroll=ft.ScrollMode.AUTO, height=300) + else: + self.emails_container.content = ft.Column([table], scroll=ft.ScrollMode.AUTO, height=300) + + self.mark_complete("mailbox", "success") + except Exception as e: + self.emails_container.content = ft.Text(f"Error: {e}", color=COLOR_ERROR) + self.mark_complete("mailbox", "error") + finally: + if self.page: + self.emails_container.update() + + def fetch_calendar(self): + try: + data = run_calendar_telemetry_pipeline(self.client, self.secret, self.tenant) + + rooms_err = data.get("RoomsError") + devs_err = data.get("DevicesError") + rooms_count = data.get("RoomsCount", 0) + equip_count = data.get("EquipmentCount", 0) + + if rooms_err and devs_err: + res_val = rooms_err + else: + r_str = "Error" if rooms_err else str(rooms_count) + e_str = "Error" if devs_err else str(equip_count) + tot = "Error" if (rooms_err or devs_err) else str(rooms_count + equip_count) + res_val = f"Total: {tot} ({r_str} Rooms, {e_str} Equipment)" + + reserve_val = data.get("CanUsersReserveRooms") + if isinstance(reserve_val, bool): + reserve_val = "Yes" if reserve_val else "No" + + att_val = data.get("CanShareAttachments") + if isinstance(att_val, bool): + attachments_val = "Yes" if att_val else "No" + else: + attachments_val = att_val + + rows_data = [ + ("Room & Resource Reservation", reserve_val), + ("Calendar Resources", res_val), + ("Integrated Calendar Apps", data.get("IntegratedCalendarApps") or "None found"), + ("Resource Naming Convention", data.get("NamingConvention") or "None found"), + ("Calendar Attachments Enabled", attachments_val), + ] + + rows = [ft.DataRow(cells=[ + ft.DataCell(ft.Text(str(r[0]), weight=ft.FontWeight.BOLD)), + ft.DataCell(ft.Text(str(r[1]))) + ]) for r in rows_data] + + table = ft.DataTable( + columns=[ + ft.DataColumn(ft.Text("Calendar Configuration / Metric", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Value / Configuration", weight=ft.FontWeight.BOLD)), + ], + rows=rows, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + border_radius=8, + heading_row_color=COLOR_TONAL_BG, + ) + + # Check for warnings + if data.get("powershell_error"): + self.calendar_container.content = ft.Column([ + ft.Text(f"⚠️ Warning: Exchange PowerShell query failed: {data['powershell_error']}", color=COLOR_WARNING, size=13), + table + ], scroll=ft.ScrollMode.AUTO, height=300) + else: + self.calendar_container.content = ft.Column([table], scroll=ft.ScrollMode.AUTO, height=300) + + self.mark_complete("calendar", "success") + except Exception as e: + self.calendar_container.content = ft.Text(f"Error: {e}", color=COLOR_ERROR) + self.mark_complete("calendar", "error") + finally: + if self.page: + self.calendar_container.update() + + def fetch_sharepoint(self): + try: + data = run_sharepoint_pipeline(self.client, self.secret, self.tenant) + rows_data = [ + ("Total Sites Count", f"{data.get('total_sites', 0):,} Sites"), + ("Total Storage Used", data.get("total_storage_formatted", "0.00 Bytes")), + ("Total Files Stored", f"{data.get('total_files', 0):,} Files"), + ("Active Files Count", f"{data.get('active_files', 0):,} Files ({data.get('active_files_pct', 0.0):.1f}%)") + ] + rows = [ft.DataRow(cells=[ + ft.DataCell(ft.Text(str(r[0]), weight=ft.FontWeight.BOLD)), + ft.DataCell(ft.Text(str(r[1]))) + ]) for r in rows_data] + + table = ft.DataTable( + columns=[ + ft.DataColumn(ft.Text("SharePoint Site Metric Description", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Value / Measurement", weight=ft.FontWeight.BOLD)), + ], + rows=rows, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + border_radius=8, + heading_row_color=COLOR_TONAL_BG, + ) + self.sharepoint_container.content = ft.Column([table], scroll=ft.ScrollMode.AUTO, height=200) + self.mark_complete("sharepoint", "success") + except Exception as e: + self.sharepoint_container.content = ft.Text(f"Error: {e}", color=COLOR_ERROR) + self.mark_complete("sharepoint", "error") + finally: + if self.page: + self.sharepoint_container.update() + + def fetch_onedrive(self): + try: + data = run_onedrive_pipeline(self.client, self.secret, self.tenant) + rows_data = [ + ("Total Accounts Count", f"{data.get('total_accounts', 0):,} Accounts"), + ("Total Storage Used", data.get("total_storage_formatted", "0.00 Bytes")), + ("Total Files Stored", f"{data.get('total_files', 0):,} Files"), + ("Active Files Count", f"{data.get('active_files', 0):,} Files ({data.get('active_files_pct', 0.0):.1f}%)"), + ("Users with Synced Files", f"{data.get('sync_users', 0):,} Users ({data.get('sync_users_pct', 0.0):.1f}%)"), + ("OneNote Active Users", f"{data.get('onenote_users', 0):,} Users") + ] + rows = [ft.DataRow(cells=[ + ft.DataCell(ft.Text(str(r[0]), weight=ft.FontWeight.BOLD)), + ft.DataCell(ft.Text(str(r[1]))) + ]) for r in rows_data] + + table = ft.DataTable( + columns=[ + ft.DataColumn(ft.Text("OneDrive Metric Description", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Value / Measurement", weight=ft.FontWeight.BOLD)), + ], + rows=rows, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + border_radius=8, + heading_row_color=COLOR_TONAL_BG, + ) + self.onedrive_container.content = ft.Column([table], scroll=ft.ScrollMode.AUTO, height=250) + self.mark_complete("onedrive", "success") + except Exception as e: + self.onedrive_container.content = ft.Text(f"Error: {e}", color=COLOR_ERROR) + self.mark_complete("onedrive", "error") + finally: + if self.page: + self.onedrive_container.update() + + def fetch_labels(self): + try: + from telemetry.data_security_governance import fetch_sensitivity_labels_data + res = fetch_sensitivity_labels_data(self.client, self.secret, self.tenant) + labels = res.get("labels") + err = res.get("error") + + # Populate Sensitivity Labels pagination data + self.flattened_labels = [] + if err: + self.labels_section.content_container.content = ft.Text(f"Error loading labels: {err}", color=COLOR_ERROR) + self.labels_pagination_row.visible = False + self.mark_complete("labels", "error") + elif not labels: + self.labels_section.content_container.content = ft.Text("No Sensitivity Labels configured in this tenant.", color=COLOR_TEXT_SUB) + self.labels_pagination_row.visible = False + self.mark_complete("labels", "success") + else: + for parent in labels: + self.flattened_labels.append({ + "name": parent.get("name", "N/A"), + "description": parent.get("description", "") or parent.get("toolTip", "") or "N/A", + "hasProtection": parent.get("hasProtection", False), + "applicationMode": parent.get("applicationMode", "N/A") or "N/A", + "priority": parent.get("priority", 0), + "applicableTo": parent.get("applicableTo", ""), + "isEnabled": parent.get("isEnabled", True), + "is_sublabel": False + }) + sublabels = parent.get("sublabels", []) + if sublabels: + sublabels_sorted = sorted(sublabels, key=lambda x: x.get("priority", 0), reverse=True) + for sub in sublabels_sorted: + self.flattened_labels.append({ + "name": f" ↳ {sub.get('name', 'N/A')}", + "description": sub.get("description", "") or sub.get("toolTip", "") or "N/A", + "hasProtection": sub.get("hasProtection", False), + "applicationMode": sub.get("applicationMode", "N/A") or "N/A", + "priority": sub.get("priority", 0), + "applicableTo": sub.get("applicableTo", ""), + "isEnabled": sub.get("isEnabled", True), + "is_sublabel": True + }) + self.current_labels_page = 0 + self.render_labels_page() + self.mark_complete("labels", "success") + except Exception as e: + self.set_error(self.labels_section, str(e)) + self.labels_pagination_row.visible = False + self.mark_complete("labels", "error") + finally: + self.clear_loading(self.labels_section) + if self.page: + self.labels_section.update() + + def fetch_retention(self): + try: + from telemetry.data_security_governance import fetch_retention_policies_data + res = fetch_retention_policies_data(self.client, self.secret, self.tenant) + policies = res.get("policies") + err = res.get("error") + + # Populate Retention Policies + if err: + msg = err + if "powershell" in err.lower() or "pwsh" in err.lower(): + msg = "PowerShell Core ('pwsh') is not installed or configured on this machine." + elif "exchangeonlinemanagement" in err.lower(): + msg = "ExchangeOnlineManagement PowerShell module is missing." + self.retention_section.content_container.content = ft.Text(f"Error loading policies: {msg}", color=COLOR_ERROR) + self.mark_complete("retention", "error") + elif not policies: + self.retention_section.content_container.content = ft.Text("No Retention Compliance Policies found.", color=COLOR_TEXT_SUB) + self.mark_complete("retention", "success") + else: + policies_list = policies if isinstance(policies, list) else [policies] + rows = [] + for policy in policies_list: + duration_val = str(policy.get("Duration", "N/A")) + duration_str = duration_val + if duration_val.lower() == "unlimited": + duration_str = "Keep Forever" + elif duration_val.isdigit(): + days = int(duration_val) + if days >= 365: + years = days / 365.0 + duration_str = f"{int(years)} Years ({days} days)" if years.is_integer() else f"{years:.1f} Years ({days} days)" + else: + duration_str = f"{days} days" + + trigger_val = policy.get("RetentionTrigger", "N/A") + if trigger_val and trigger_val != "N/A": + trigger_map = {"DateCreated": "created date", "DateModified": "last modified date", "DateLabeled": "labeled date"} + duration_str += f"\n(from {trigger_map.get(trigger_val, trigger_val)})" + + enabled_val = policy.get("Enabled", True) + is_enabled = enabled_val.lower() == "true" if isinstance(enabled_val, str) else bool(enabled_val) + status_str = "🟢 Enabled" if is_enabled else "🔴 Disabled" + + rows.append(ft.DataRow(cells=[ + ft.DataCell(ft.Column([ + ft.Text(policy.get("Name", "N/A"), weight=ft.FontWeight.BOLD), + ft.Text(policy.get("Comment", ""), size=11, color=COLOR_TEXT_SUB) if policy.get("Comment") else ft.Container() + ], alignment=ft.MainAxisAlignment.CENTER)), + ft.DataCell(ft.Text(policy.get("Workload", "N/A"))), + ft.DataCell(ft.Text(duration_str)), + ft.DataCell(ft.Text(policy.get("DistributionStatus", "Success"))), + ft.DataCell(ft.Text(status_str)) + ])) + + table = ft.DataTable( + columns=[ + ft.DataColumn(ft.Text("Policy Name", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Workloads", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Duration", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Distribution", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Status", weight=ft.FontWeight.BOLD)), + ], + rows=rows, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + border_radius=8, + heading_row_color=COLOR_TONAL_BG, + ) + self.retention_section.content_container.content = ft.Column([table], scroll=ft.ScrollMode.AUTO, height=300) + self.mark_complete("retention", "success") + except Exception as e: + self.set_error(self.retention_section, str(e)) + self.mark_complete("retention", "error") + finally: + self.clear_loading(self.retention_section) + if self.page: + self.retention_section.update() + + def render_labels_page(self): + total_items = len(self.flattened_labels) + total_pages = (total_items + self.labels_per_page - 1) // self.labels_per_page + if total_pages < 1: + total_pages = 1 + + start_idx = self.current_labels_page * self.labels_per_page + end_idx = min(start_idx + self.labels_per_page, total_items) + page_items = self.flattened_labels[start_idx:end_idx] + + rows = [] + for item in page_items: + protection = "🛡️ Yes" if item["hasProtection"] else "🔓 No" + status_str = "🟢 Enabled" if item["isEnabled"] else "🔴 Disabled" + + name_color = COLOR_TEXT_MAIN if not item["is_sublabel"] else COLOR_TEXT_SUB + name_weight = ft.FontWeight.BOLD if not item["is_sublabel"] else ft.FontWeight.NORMAL + + rows.append(ft.DataRow(cells=[ + ft.DataCell(ft.Text(item["name"], weight=name_weight, color=name_color)), + ft.DataCell(ft.Text(item["description"])), + ft.DataCell(ft.Text(protection)), + ft.DataCell(ft.Text(str(item["applicationMode"]).capitalize())), + ft.DataCell(ft.Text(str(item["priority"]))), + ft.DataCell(ft.Text(", ".join([x.capitalize() for x in item["applicableTo"].split(",") if x.strip()]) or "N/A")), + ft.DataCell(ft.Text(status_str)) + ])) + + table = ft.DataTable( + columns=[ + ft.DataColumn(ft.Text("Sensitivity Label", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Description", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Protection", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Mode", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Priority", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Applicable Targets", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Status", weight=ft.FontWeight.BOLD)), + ], + rows=rows, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + border_radius=8, + heading_row_color=COLOR_TONAL_BG, + ) + self.labels_section.content_container.content = ft.Column([table], scroll=ft.ScrollMode.AUTO, height=350) + + # Update Pagination Control status + self.labels_pagination_info.value = f"Page {self.current_labels_page + 1} of {total_pages}" + self.labels_prev_btn.disabled = (self.current_labels_page <= 0) + self.labels_next_btn.disabled = (self.current_labels_page >= total_pages - 1) + self.labels_pagination_row.visible = (total_items > self.labels_per_page) + + if self.page: + self.labels_section.update() + self.labels_pagination_row.update() + + def handle_labels_prev(self, e): + if self.current_labels_page > 0: + self.current_labels_page -= 1 + self.render_labels_page() + + def handle_labels_next(self, e): + total_items = len(self.flattened_labels) + total_pages = (total_items + self.labels_per_page - 1) // self.labels_per_page + if self.current_labels_page < total_pages - 1: + self.current_labels_page += 1 + self.render_labels_page() + + def fetch_pa(self): + try: + scanner = PowerAutomateScanner(self.tenant, self.client, self.secret) + results = scanner.scan_flows() + if not results: + self.pa_section.content_container.content = ft.Text("No Power Automate data found.", color=COLOR_TEXT_SUB) + self.export_pa_btn.disabled = True + else: + total_envs = results.get("total_environments", 0) + counts = results.get("counts", {}) + total_flows = counts.get("Cloud Flows", 0) + counts.get("Desktop Flows", 0) + premium_conns = results.get("premium_connectors", []) + custom_conns = results.get("custom_connectors", []) + self.last_complex_flows = results.get("complex_logic_flows", []) + + prem_str = ", ".join(premium_conns) if premium_conns else "0" + cust_str = ", ".join(custom_conns) if custom_conns else "0" + + rows_data = [ + ("Total Environments Scanned", str(total_envs)), + ("Total Flows (Active + Inactive)", str(total_flows)), + ("Premium Connectors In Use", prem_str), + ("Custom Connectors In Use", cust_str), + ] + rows = [ft.DataRow(cells=[ + ft.DataCell(ft.Text(str(r[0]), weight=ft.FontWeight.BOLD)), + ft.DataCell(ft.Text(str(r[1]))) + ]) for r in rows_data] + + table = ft.DataTable( + columns=[ + ft.DataColumn(ft.Text("Metric", weight=ft.FontWeight.BOLD)), + ft.DataColumn(ft.Text("Value", weight=ft.FontWeight.BOLD)), + ], + rows=rows, + border=ft.Border.all(1, COLOR_OUTLINE_LIGHT), + border_radius=8, + heading_row_color=COLOR_TONAL_BG, + ) + self.pa_section.content_container.content = table + self.export_pa_btn.disabled = (len(self.last_complex_flows) == 0) + + self.mark_complete("pa", "success") + except Exception as e: + self.set_error(self.pa_section, str(e)) + self.export_pa_btn.disabled = True + self.mark_complete("pa", "error") + finally: + self.clear_loading(self.pa_section) + if self.page: + self.pa_section.update() + self.export_pa_btn.update() + + # --- CSV Export Handlers --- + + def handle_export_skus(self, e): + def on_save_result(save_event: ft.FilePickerResultEvent): + if save_event.path: + try: + headers = ["SKU Part Number", "Units", "Consumed Units", "Included Service Plans", "Applies To"] + rows = [] + for item in self.last_licenses_items: + sku_name = item.get("skuPartNumber", "UNKNOWN_SKU") + prepaid = item.get("prepaidUnits", {}) + enabled_units = prepaid.get("enabled", 0) + warn_units = prepaid.get("warning", 0) + susp_units = prepaid.get("suspended", 0) + + prepaid_str = f"Enabled: {enabled_units:,}" + if warn_units > 0: prepaid_str += f"\nWarn: {warn_units:,}" + if susp_units > 0: prepaid_str += f"\nSusp: {susp_units:,}" + consumed_str = f"{item.get('consumedUnits', 0):,}" + + plans = item.get("servicePlans", []) + + if not plans: + rows.append([sku_name, prepaid_str, consumed_str, "None designated.", "-"]) + else: + for idx, p in enumerate(plans): + p_name = p.get("servicePlanName", "UnnamedPlan") + p_scope = p.get("appliesTo", "Unknown") + if idx == 0: + rows.append([sku_name, prepaid_str, consumed_str, p_name, p_scope]) + else: + rows.append(["", "", "", p_name, p_scope]) + + with open(save_event.path, 'w', newline='', encoding='utf-8') as csvfile: + writer = csv.writer(csvfile) + writer.writerow(headers) + writer.writerows(rows) + + except Exception as ex: + print(f"Failed to export SKUs: {ex}") + + picker = ft.FilePicker(on_result=on_save_result) + e.page.overlay.append(picker) + e.page.update() + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + picker.save_file(file_name=f"licenses_inventory_{ts}.csv") + + def handle_export_pa(self, e): + def on_save_result(save_event: ft.FilePickerResultEvent): + if save_event.path: + try: + headers = ["Environment", "Name", "Type", "Tier", "Active", "Reason"] + rows = [] + for flow in self.last_complex_flows: + rows.append([ + flow.get("Environment"), + flow.get("Name"), + flow.get("Type"), + flow.get("Tier"), + flow.get("Active"), + flow.get("Reason") + ]) + + with open(save_event.path, 'w', newline='', encoding='utf-8') as csvfile: + writer = csv.writer(csvfile) + writer.writerow(headers) + writer.writerows(rows) + except Exception as ex: + print(f"Failed to export complex flows: {ex}") + + picker = ft.FilePicker(on_result=on_save_result) + e.page.overlay.append(picker) + e.page.update() + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + picker.save_file(file_name=f"complex_flows_{ts}.csv") diff --git a/flet_app/main.py b/flet_app/main.py new file mode 100644 index 00000000..076184d2 --- /dev/null +++ b/flet_app/main.py @@ -0,0 +1,113 @@ +import flet as ft +import sys +import os + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from flet_app.styles import get_theme, COLOR_BACKGROUND +from flet_app.auth_view import AuthView +from flet_app.cert_instructions_view import CertInstructionsView +from flet_app.dashboard import DashboardView +from core.cert_auth import check_certificate_exists, generate_certificate, load_certificate + +def main(page: ft.Page): + page.title = "Deal Assistant (Flet)" + page.theme_mode = ft.ThemeMode.LIGHT + page.theme = get_theme() + page.bgcolor = COLOR_BACKGROUND + page.padding = 20 + + # Store session variables + page.session.store.set("tenant", "") + page.session.store.set("client", "") + page.session.store.set("secret", "") + + def show_error_dialog(title, message): + def close_dialog(e): + dialog.open = False + page.update() + dialog = ft.AlertDialog( + title=ft.Text(title, weight=ft.FontWeight.BOLD), + content=ft.Text(message), + actions=[ft.TextButton("Close", on_click=close_dialog)], + actions_alignment=ft.MainAxisAlignment.END, + ) + page.overlay.append(dialog) + dialog.open = True + page.update() + + def show_auth(): + page.controls.clear() + page.add(AuthView(on_connect_clicked=handle_connect)) + page.update() + + def show_dashboard(): + tenant = page.session.store.get("tenant") + client = page.session.store.get("client") + secret = page.session.store.get("secret") + page.controls.clear() + page.add(DashboardView(tenant, client, secret, on_disconnect=handle_disconnect)) + page.update() + + def show_cert_instructions(pem_path): + client = page.session.store.get("client") + page.controls.clear() + page.add(CertInstructionsView(pem_path=pem_path, client_id=client, on_continue=handle_cert_continue)) + page.update() + + def handle_connect(tenant, client, secret): + page.session.store.set("tenant", tenant) + page.session.store.set("client", client) + page.session.store.set("secret", secret) + + if check_certificate_exists(tenant_id=tenant, client_id=client): + try: + # Decrypt the PFX certificate using the client secret + load_certificate(secret, tenant_id=tenant, client_id=client) + show_dashboard() + except Exception as e: + show_error_dialog( + "Certificate Decryption Error", + f"Unable to unlock certificate with Client Secret. Proceeding with standard Client Secret authentication fallback.\n\nError: {e}" + ) + show_dashboard() + else: + try: + # Generate new certificate and pfx encrypted with the client secret + pem_path, _ = generate_certificate(secret, tenant_id=tenant, client_id=client) + # Show instructions UI + show_cert_instructions(pem_path) + except Exception as e: + show_error_dialog( + "Certificate Generation Error", + f"Unable to generate certificate. Proceeding with standard Client Secret authentication fallback.\n\nError: {e}" + ) + show_dashboard() + + def handle_cert_continue(): + tenant = page.session.store.get("tenant") + client = page.session.store.get("client") + secret = page.session.store.get("secret") + try: + load_certificate(secret, tenant_id=tenant, client_id=client) + except Exception as e: + show_error_dialog( + "Certificate Verification Error", + f"Unable to verify certificate. Proceeding with standard Client Secret authentication fallback.\n\nError: {e}" + ) + show_dashboard() + + def handle_disconnect(): + page.session.store.set("tenant", "") + page.session.store.set("client", "") + page.session.store.set("secret", "") + show_auth() + + # Start app on auth page + show_auth() + +if __name__ == "__main__": + # Ensure matplotlib backend doesn't crash Flet if graph operations are pulled + import matplotlib + matplotlib.use("Agg") + + ft.run(main) diff --git a/flet_app/sidebar.py b/flet_app/sidebar.py new file mode 100644 index 00000000..841ef69e --- /dev/null +++ b/flet_app/sidebar.py @@ -0,0 +1,61 @@ +import flet as ft +from flet_app.styles import * + +class Sidebar(ft.Container): + def __init__(self, on_disconnect): + super().__init__() + self.on_disconnect = on_disconnect + + self.width = 300 + self.bgcolor = COLOR_SURFACE + self.border_radius = 12 + self.border = ft.Border.all(1, COLOR_OUTLINE_LIGHT) + self.padding = 20 + + menu_items = [ + ("Usage and adoption", ft.Icons.BAR_CHART, True), + ("Workforce analysis", ft.Icons.PEOPLE, False), + ("Cost savings plan", ft.Icons.ATTACH_MONEY, False), + ("Migration planner", ft.Icons.ROCKET_LAUNCH, False) + ] + + self.menu_column = ft.Column(spacing=10) + + for label, icon, is_active in menu_items: + bg = COLOR_TONAL_BG if is_active else "transparent" + text_col = COLOR_PRIMARY if is_active else COLOR_TEXT_SUB + weight = ft.FontWeight.BOLD if is_active else ft.FontWeight.NORMAL + + btn = ft.Container( + content=ft.Row([ + ft.Icon(icon, color=text_col, size=20), + ft.Text(label, color=text_col, weight=weight, size=14) + ]), + bgcolor=bg, + padding=ft.Padding.symmetric(horizontal=15, vertical=12), + border_radius=8, + ink=True if not is_active else False, + ) + self.menu_column.controls.append(btn) + + self.content = ft.Column( + controls=[ + ft.Row([ + ft.Text("🤝", size=24), + ft.Text("Deal Assistant", size=18, weight=ft.FontWeight.BOLD, color=COLOR_TEXT_MAIN) + ], alignment=ft.MainAxisAlignment.START), + ft.Divider(height=30, color="transparent"), + self.menu_column, + ft.Container(expand=True), # Spacer + ft.Container( + content=ft.Row([ + ft.Icon(ft.Icons.LOGOUT, color=COLOR_ERROR, size=20), + ft.Text("Disconnect", color=COLOR_ERROR, weight=ft.FontWeight.W_500, size=14) + ]), + padding=ft.Padding.symmetric(horizontal=15, vertical=12), + border_radius=8, + ink=True, + on_click=lambda _: self.on_disconnect() + ) + ] + ) diff --git a/flet_app/styles.py b/flet_app/styles.py new file mode 100644 index 00000000..9381c155 --- /dev/null +++ b/flet_app/styles.py @@ -0,0 +1,27 @@ +import flet as ft + +COLOR_PRIMARY = "#1E3A8A" +COLOR_PRIMARY_HOVER = "#172554" +COLOR_SECONDARY = "#3B82F6" +COLOR_SECONDARY_HOVER = "#DBEAFE" +COLOR_BACKGROUND = "#F8FAFC" +COLOR_SURFACE = "#FFFFFF" +COLOR_SURFACE_VARIANT = "#F1F5F9" +COLOR_ERROR = "#DC2626" +COLOR_SUCCESS = "#10B981" +COLOR_WARNING = "#F59E0B" +COLOR_TEXT_MAIN = "#0F172A" +COLOR_TEXT_SUB = "#64748B" +COLOR_OUTLINE = "#CBD5E1" +COLOR_OUTLINE_LIGHT = "#E2E8F0" +COLOR_TONAL_BG = "#EFF6FF" +COLOR_TONAL_TEXT = "#1E40AF" + +def get_theme(): + return ft.Theme( + color_scheme=ft.ColorScheme( + primary=COLOR_PRIMARY, + surface=COLOR_SURFACE, + error=COLOR_ERROR, + ) + ) diff --git a/telemetry/README.md b/telemetry/README.md new file mode 100644 index 00000000..cc474efe --- /dev/null +++ b/telemetry/README.md @@ -0,0 +1,79 @@ +# Telemetry Module + +## Prerequisites + +The certificate authentication flow, reports parsing, and user interface require the following Python libraries: +* `customtkinter` +* `requests` +* `pandas` +* `psutil` +* `matplotlib` +* `cryptography` +* `msal` +* `google-genai` +* `scikit-learn` + +You can install them via pip: +```bash +pip install customtkinter requests pandas psutil matplotlib cryptography msal google-genai scikit-learn +``` + +## Architecture & Optimizations +For large tenant scopes (e.g., millions of records or 100K+ flows), this module utilizes aggressive disk-caching mechanisms out-of-the-box, ensuring the application remains lightweight on RAM: +- **SQLite UI Pagination**: Data grids are lazily fetched from `sqlite3` temp databases rather than hoarding UI components in Python lists. +- **Disk Streaming Pipelines**: Complex parsing arrays are continuously streamed to local `.jsonl` temp files and pushed natively to `pandas.DataFrame` chunking logic for exports. +- **Lazy Garbage Collection**: Core navigation state handles `gc.collect()` passively between UI tab cycles. + + +## Setup & Execution + +### 1. Entra ID Permissions (Tenant-Wide Cloud Flows) + +1. Navigate to the [Microsoft Entra ID Portal](https://www.google.com/search?q=https://entra.microsoft.com/) > **Roles and administrators**. +2. Assign the **Power Platform Administrator** role to your App Registration. + +### 2. Dataverse Permissions (Desktop Flows) + +*Perform this in each environment where you need to scan Desktop Flows:* + +1. Navigate to the [Power Platform Admin Center](https://www.google.com/search?q=https://admin.powerplatform.microsoft.com/) > **Environments** > [Select Environment] > **Settings**. +2. Under **Users + permissions** > **Application users**, click **+ New app user**. +3. Add your App Registration and assign it the **System Administrator** role. + +### 3. Certificate Setup (Hybrid Authentication) + +The telemetry planner uses local certificate-based authentication for connecting securely to Microsoft APIs: + +1. When running the Telemetry tool, it checks for a directory named `certificate/{tenantId}_{clientId}` containing `passkey.pfx` under the root of `migration-planner`. +2. If this file does not exist, the app automatically generates a self-signed public certificate (`certificate.pem`) and an encrypted private key bundle (`passkey.pfx`) under the dynamic `certificate/{tenantId}_{clientId}` directory using the provided Client Secret as the password. +3. You will be prompted in the UI to upload `certificate.pem` to Microsoft Entra ID: + - Navigate to the **Microsoft Entra ID Portal** > **App registrations** > [Select your Application]. + - Click **Certificates & secrets** > **Certificates** tab > **Upload certificate**. + - Select and upload the generated `certificate.pem` file. +4. Click **Continue** in the application interface to complete the connection flow. + +### 4. Entra ID App & PowerShell Permissions (Calendar & Mailbox Telemetry) + +For the core telemetry scanners (Calendar Telemetry, Active Users, Mailbox/SharePoint Usage, etc.) to query successfully: + +#### A. Microsoft Graph API Permissions (Application Scopes) +Ensure the following **Application** API permissions are granted and admin-consented in your App Registration: +- `Place.Read.All`: Used to list meeting rooms and resource device counts. +- `User.Read.All`: Used to read user directory identities to aggregate settings. +- `Calendars.ReadBasic.All`: Used to audit organizational calendar permissions. +- `Reports.Read.All`: Used to retrieve active user trends and mailbox/SharePoint usage reports. +- `Directory.Read.All`: Used to read tenant organization configuration data. + +#### B. Exchange Online PowerShell Roles +The certificate-based PowerShell client requires administrative roles to read Exchange policies (OWA, default apps, sharing policies). +In the **Microsoft Entra ID Portal** > **Roles and administrators**, assign one of the following directory roles to your App Registration: +- **Global Reader** (Recommended, read-only) +- **Exchange Administrator** + +## Logging +Logs are appended to `telemetry/logs/power_automate_log.txt`. + +## Troubleshooting +If you encounter a `400 Client Error: Bad Request` when querying the workflows endpoint, please verify: +1. The App is properly allowlisted in the Power Platform Admin Center. +2. The Environment URL is correct and accessible. \ No newline at end of file diff --git a/telemetry/active_users_usage.py b/telemetry/active_users_usage.py new file mode 100644 index 00000000..7c22d941 --- /dev/null +++ b/telemetry/active_users_usage.py @@ -0,0 +1,713 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Modular O365 Active Users usage telemetry scanners, aggregation pipelines, and visual interfaces.""" + +import os +import sys +import logging +import threading +import pandas as pd +from datetime import datetime, date +from typing import Any, List +import customtkinter as ctk + +# Import unified core service layer +from core.graph.client import GraphClient +from core.graph.reports import ReportsService + +# Safely import matplotlib to embed plots in Tkinter +try: + from matplotlib.figure import Figure + from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg + MATPLOTLIB_AVAILABLE = True +except ImportError: + MATPLOTLIB_AVAILABLE = False + +# Bind to the async logger initialized in m365_telemetry.py +usage_logger = logging.getLogger("M365TelemetryAsyncLogger") + +# Import shared styles +from telemetry.styles import * + + +# ================================================================================= +# PIPELINE LOGIC / PROCESSORS +# ================================================================================= + +def _get_reports_service(client_id, client_secret, tenant_id) -> tuple[GraphClient, ReportsService]: + """Helper to instantiate GraphClient/ReportsService and manage credentials slots.""" + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=2, + retries=5, + backoff=2 + ) + client.authenticate() + return client, ReportsService(client) + + +def process_active_user_detail(filepath): + """Streams the downloaded CSV and calculates usage counters over 30, 90, and 180 days.""" + usage_logger.info(f"Processing O365 file: {os.path.basename(filepath)}") + + if not os.path.exists(filepath): + usage_logger.error(f"Error: Could not find the file {filepath} to process.") + raise FileNotFoundError(f"Report file {os.path.basename(filepath)} not found. Download may have failed.") + + current_date = pd.Timestamp.today().normalize() + + headers = pd.read_csv(filepath, nrows=0).columns.tolist() + expected = [ + "Has Exchange License", "Exchange Last Activity Date", + "Has OneDrive License", "OneDrive Last Activity Date", + "Has SharePoint License", "SharePoint Last Activity Date", + "Has Teams License", "Teams Last Activity Date" + ] + cols = [c for c in expected if c in headers] + + exchange_online_usage = [0, 0, 0] + onedrive_usage = [0, 0, 0] + sharepoint_usage = [0, 0, 0] + teams_usage = [0, 0, 0] + + def process_chunk_col(chunk, has_license_col, date_col): + if has_license_col not in chunk.columns or date_col not in chunk.columns: + return [0, 0, 0] + mask = chunk[has_license_col].astype(str).str.strip().str.upper() == "TRUE" + dates_series = pd.to_datetime(chunk.loc[mask, date_col], errors='coerce') + days_diff = (current_date - dates_series).dt.days + d180 = int((days_diff < 180).sum()) + d90 = int((days_diff < 90).sum()) + d30 = int((days_diff < 30).sum()) + return [d30, d90, d180] + + for chunk in pd.read_csv(filepath, usecols=cols, chunksize=10000, encoding="utf-8-sig"): + e_chunk = process_chunk_col(chunk, "Has Exchange License", "Exchange Last Activity Date") + exchange_online_usage = [x + y for x, y in zip(exchange_online_usage, e_chunk)] + + od_chunk = process_chunk_col(chunk, "Has OneDrive License", "OneDrive Last Activity Date") + onedrive_usage = [x + y for x, y in zip(onedrive_usage, od_chunk)] + + sp_chunk = process_chunk_col(chunk, "Has SharePoint License", "SharePoint Last Activity Date") + sharepoint_usage = [x + y for x, y in zip(sharepoint_usage, sp_chunk)] + + t_chunk = process_chunk_col(chunk, "Has Teams License", "Teams Last Activity Date") + teams_usage = [x + y for x, y in zip(teams_usage, t_chunk)] + + usage_logger.info("Successfully processed O365 active user data in chunks.") + return [ + ("Exchange Online", exchange_online_usage[0], exchange_online_usage[1], exchange_online_usage[2]), + ("OneDrive", onedrive_usage[0], onedrive_usage[1], onedrive_usage[2]), + ("SharePoint", sharepoint_usage[0], sharepoint_usage[1], sharepoint_usage[2]), + ("Teams", teams_usage[0], teams_usage[1], teams_usage[2]) + ] + + +def process_active_user_counts(filepath): + """Parses chronological usage data for plotting.""" + usage_logger.info(f"Processing O365 Counts file: {os.path.basename(filepath)}") + if not os.path.exists(filepath): + usage_logger.error(f"Error: Could not find the file {filepath} to process.") + raise FileNotFoundError(f"Report file {os.path.basename(filepath)} not found.") + + headers = pd.read_csv(filepath, nrows=0).columns.tolist() + expected = ["Report Date", "Office 365", "Exchange", "OneDrive", "SharePoint", "Teams"] + cols = [c for c in expected if c in headers] + dates = [] + office365 = [] + exchange = [] + onedrive = [] + sharepoint = [] + teams = [] + + for chunk in pd.read_csv(filepath, usecols=cols, chunksize=10000, encoding="utf-8-sig"): + if "Report Date" in chunk.columns: + chunk = chunk.sort_values(by="Report Date").fillna(0) + dates.extend(chunk["Report Date"].astype(str).tolist()) + else: + dates.extend([""] * len(chunk)) + + def extract_col(col_name): + if col_name in chunk.columns: + return pd.to_numeric(chunk[col_name], errors='coerce').fillna(0).astype(int).tolist() + return [0] * len(chunk) + + office365.extend(extract_col("Office 365")) + exchange.extend(extract_col("Exchange")) + onedrive.extend(extract_col("OneDrive")) + sharepoint.extend(extract_col("SharePoint")) + teams.extend(extract_col("Teams")) + + usage_logger.info("Successfully processed O365 active user counts data.") + return { + "dates": dates, + "office365": office365, + "exchange": exchange, + "onedrive": onedrive, + "sharepoint": sharepoint, + "teams": teams + } + + +def process_m365_app_user_detail(filepath): + """Streams the downloaded CSV and calculates usage counters.""" + usage_logger.info(f"Processing M365 App file: {os.path.basename(filepath)}") + + if not os.path.exists(filepath): + usage_logger.error(f"Error: Could not find the file {filepath} to process.") + raise FileNotFoundError(f"Report file {os.path.basename(filepath)} not found. Download may have failed.") + + columns_to_track = [ + "Windows", "Mac", "Mobile", "Web", "Outlook", "Word", "Excel", + "PowerPoint", "OneNote", "Teams", "Outlook (Windows)", "Word (Windows)", + "Excel (Windows)", "PowerPoint (Windows)", "OneNote (Windows)", + "Teams (Windows)", "Outlook (Mac)", "Word (Mac)", "Excel (Mac)", + "PowerPoint (Mac)", "OneNote (Mac)", "Teams (Mac)", "Outlook (Mobile)", + "Word (Mobile)", "Excel (Mobile)", "PowerPoint (Mobile)", + "OneNote (Mobile)", "Teams (Mobile)", "Outlook (Web)", "Word (Web)", + "Excel (Web)", "PowerPoint (Web)", "OneNote (Web)", "Teams (Web)" + ] + + headers = pd.read_csv(filepath, nrows=0).columns.tolist() + cols = [c for c in columns_to_track if c in headers] + + counters = {col: 0 for col in columns_to_track} + + for chunk in pd.read_csv(filepath, usecols=cols, chunksize=10000, encoding="utf-8-sig"): + for col in columns_to_track: + if col in chunk.columns: + col_series = chunk[col].astype(str).str.strip().str.lower() + count = int(col_series.isin(["yes", "true"]).sum()) + counters[col] += count + + usage_logger.info("Successfully processed M365 App user data in chunks.") + return [(col, count) for col, count in counters.items()] + + +def run_o365_pipeline(client_id, client_secret, tenant_id): + """Pipeline specifically for O365 Active User Data.""" + usage_logger.info("Starting isolated O365 Pipeline...") + client, service = _get_reports_service(client_id, client_secret, tenant_id) + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + reports_dir = os.path.join(script_dir, "reports", f"{tenant_id}_{client_id}") + + service.download_o365_active_user_detail(reports_dir) + client.close() + + return process_active_user_detail(os.path.join(reports_dir, "Office365ActiveUserDetail(180d).csv")) + + +def run_o365_trend_pipeline(client_id, client_secret, tenant_id): + """Pipeline specifically for O365 Trend Data.""" + try: + usage_logger.info("Starting isolated O365 Trend Pipeline...") + client, service = _get_reports_service(client_id, client_secret, tenant_id) + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + reports_dir = os.path.join(script_dir, "reports", f"{tenant_id}_{client_id}") + + service.download_o365_active_user_counts(reports_dir) + client.close() + + return process_active_user_counts(os.path.join(reports_dir, "Office365ActiveUserCounts(30d).csv")) + except Exception as e: + usage_logger.error("O365 Trend pipeline failed.", exc_info=True) + raise + + +def run_m365_pipeline(client_id, client_secret, tenant_id): + """Pipeline specifically for M365 Apps Data.""" + usage_logger.info("Starting isolated M365 Apps Pipeline...") + client, service = _get_reports_service(client_id, client_secret, tenant_id) + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + reports_dir = os.path.join(script_dir, "reports", f"{tenant_id}_{client_id}") + + service.download_m365_app_details(reports_dir) + client.close() + + return process_m365_app_user_detail(os.path.join(reports_dir, "M365AppUserDetail(180d).csv")) + + +# ================================================================================= +# MODULAR UI COMPONENTS +# ================================================================================= + +class ActiveUsersUsageFrame(ctk.CTkFrame): + """Self-contained component wrapping O365 Active Users Usage UI.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None # 'loading', 'success', 'error', None + + self.build_ui() + + def build_ui(self): + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + ctk.CTkLabel(self.inner_pad, text="O365 Active Users Usage", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(anchor="w", pady=(0, 10)) + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_OUTLINE_LIGHT, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.reset_view() + + def reset_view(self): + self.pack_forget() + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=20) + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + display_msg = "Reports telemetry permission required.\nPlease grant the 'Reports.Read.All' application permission to your App Registration in Entra ID." + + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 10)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + usage_logger.info("Active Users Usage trigger_fetch called. Spawning background worker thread...") + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=10) + self.grid_frame.pack_forget() + + self._set_state_loading("Downloading and parsing O365 Active Users reports...") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str): + if self.semaphore: + self.semaphore.acquire() + try: + o365_data = run_o365_pipeline(client_id, client_secret, tenant) + usage_logger.info("Successfully completed O365 usage data fetch.") + self.after(0, self._render_success, o365_data) + except Exception as e: + usage_logger.error("Exception caught in ActiveUsersUsage worker.", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, o365_data: list): + self.o365_data = o365_data + self.state_frame.pack_forget() + for w in self.grid_frame.winfo_children(): + w.destroy() + + self.grid_frame.pack(fill="x", expand=True) + + self.grid_frame.grid_columnconfigure(0, weight=2) + self.grid_frame.grid_columnconfigure(1, weight=1) + self.grid_frame.grid_columnconfigure(2, weight=1) + self.grid_frame.grid_columnconfigure(3, weight=1) + + headers_o365 = ["Service", "30 Days", "90 Days", "180 Days"] + for col_idx, head_text in enumerate(headers_o365): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + if not o365_data: + empty_cell = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + empty_cell.grid(row=1, column=0, columnspan=4, sticky="nsew", pady=15) + ctk.CTkLabel(empty_cell, text="No O365 usage data found.", text_color=COLOR_TEXT_SUB).pack() + else: + for r_idx, row_data in enumerate(o365_data, start=1): + bg_style = COLOR_SURFACE if r_idx % 2 == 0 else COLOR_SURFACE_VARIANT + for c_idx, val in enumerate(row_data): + cell = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + cell.grid(row=r_idx, column=c_idx, sticky="nsew", padx=0, pady=(0, 1)) + fnt = FONT_BODY_BOLD if c_idx == 0 else FONT_BODY_MEDIUM + ctk.CTkLabel(cell, text=str(val), font=fnt, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=8, anchor="nw") + + self.status = "success" + self.on_status_change() + + def _render_error(self, err_msg): + usage_logger.warning(f"Active Users Usage fetch failed: {err_msg}") + self._set_state_error(err_msg) + self.status = "error" + self.on_status_change() + + +class ActiveUsersTrendFrame(ctk.CTkFrame): + """Self-contained component wrapping O365 Active User Trend Chart and height controls.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None # 'loading', 'success', 'error', None + self.trend_data = {} + + self.build_ui() + + def build_ui(self): + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + trend_header = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + trend_header.pack(fill="x", pady=(0, 10)) + + ctk.CTkLabel(trend_header, text="O365 30-Day Active User Trend", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(side="left") + + self.trend_height_var = ctk.DoubleVar(value=400) + + slider_frame = ctk.CTkFrame(trend_header, fg_color="transparent") + slider_frame.pack(side="right") + + self.lbl_trend_height = ctk.CTkLabel(slider_frame, text="Height: 400px", font=FONT_BODY_SMALL, text_color=COLOR_TEXT_SUB) + self.lbl_trend_height.pack(side="left", padx=(0, 10)) + + self.slider_trend_height = ctk.CTkSlider( + slider_frame, from_=200, to=800, number_of_steps=60, + variable=self.trend_height_var, width=120, height=16, + command=self._on_trend_height_slider_change + ) + self.slider_trend_height.pack(side="left") + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + + self.grid_frame = ctk.CTkFrame( + self.inner_pad, fg_color=COLOR_SURFACE, + border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8, + height=400 + ) + self.grid_frame.pack_propagate(False) + + self.reset_view() + + def reset_view(self): + self.pack_forget() + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + self.trend_data = {} + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=20) + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + display_msg = "Reports telemetry permission required.\nPlease grant the 'Reports.Read.All' application permission to your App Registration in Entra ID." + + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 10)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + usage_logger.info("Active Users Trend trigger_fetch called. Spawning background worker thread...") + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=10) + self.grid_frame.pack_forget() + + self._set_state_loading("Downloading and parsing O365 Trend reports...") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str): + if self.semaphore: + self.semaphore.acquire() + try: + trend_data = run_o365_trend_pipeline(client_id, client_secret, tenant) + usage_logger.info("Successfully completed O365 trend data fetch.") + self.after(0, self._render_success, trend_data) + except Exception as e: + usage_logger.error("Exception caught in ActiveUsersTrend worker.", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, trend_data: dict): + self.trend_data = trend_data + self.state_frame.pack_forget() + for w in self.grid_frame.winfo_children(): + w.destroy() + + self.grid_frame.pack(fill="both", expand=True) + + if not MATPLOTLIB_AVAILABLE: + empty_cell = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + empty_cell.pack(fill="x", expand=True, pady=15) + ctk.CTkLabel(empty_cell, text="Matplotlib is required to render charts.\nPlease install it using 'pip install matplotlib'.", text_color=COLOR_ERROR).pack() + self.status = "error" + self.on_status_change() + return + + if not trend_data or not trend_data.get("dates"): + empty_cell = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + empty_cell.pack(fill="x", expand=True, pady=15) + ctk.CTkLabel(empty_cell, text="No O365 trend data found.", text_color=COLOR_TEXT_SUB).pack() + else: + try: + fig = Figure(figsize=(8, 4), dpi=100) + ax = fig.add_subplot(111) + fig.patch.set_facecolor(COLOR_SURFACE) + ax.set_facecolor(COLOR_SURFACE) + + dates = trend_data["dates"] + + ax.plot(dates, trend_data["office365"], marker='o', label='Office 365') + ax.plot(dates, trend_data["exchange"], marker='o', label='Exchange') + ax.plot(dates, trend_data["onedrive"], marker='o', label='OneDrive') + ax.plot(dates, trend_data["sharepoint"], marker='o', label='SharePoint') + ax.plot(dates, trend_data["teams"], marker='o', label='Teams') + + ax.set_xlabel("Date", fontsize=10, color=COLOR_TEXT_SUB) + ax.set_ylabel("Active Users", fontsize=10, color=COLOR_TEXT_SUB) + + ax.tick_params(axis='x', colors=COLOR_TEXT_SUB, rotation=45, labelsize=8) + ax.tick_params(axis='y', colors=COLOR_TEXT_SUB) + + if len(dates) > 10: + ax.set_xticks(dates[::max(1, len(dates)//10)]) + + for spine in ax.spines.values(): + spine.set_color(COLOR_OUTLINE_LIGHT) + + ax.legend(facecolor=COLOR_SURFACE, edgecolor=COLOR_OUTLINE_LIGHT, labelcolor=COLOR_TEXT_MAIN, fontsize=9) + fig.tight_layout() + + canvas = FigureCanvasTkAgg(fig, master=self.grid_frame) + canvas.draw() + canvas.get_tk_widget().pack(fill="both", expand=True, padx=10, pady=10) + except Exception as e: + usage_logger.error(f"Error drawing matplotlib plot: {e}", exc_info=True) + empty_cell = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + empty_cell.pack(fill="x", expand=True, pady=15) + ctk.CTkLabel(empty_cell, text="Failed to render trend graph (Matplotlib constraint).", text_color=COLOR_ERROR).pack() + + self.status = "success" + self.on_status_change() + + def _render_error(self, err_msg): + usage_logger.warning(f"Active Users Trend fetch failed: {err_msg}") + self._set_state_error(err_msg) + self.status = "error" + self.on_status_change() + + def _on_trend_height_slider_change(self, val): + height_val = int(val) + self.lbl_trend_height.configure(text=f"Height: {height_val}px") + self.grid_frame.configure(height=height_val) + + +class M365AppUsageFrame(ctk.CTkFrame): + """Self-contained component wrapping M365 App Usage UI.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None # 'loading', 'success', 'error', None + + self.build_ui() + + def build_ui(self): + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + ctk.CTkLabel(self.inner_pad, text="M365 App Usage (180 Days)", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(anchor="w", pady=(0, 10)) + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_OUTLINE_LIGHT, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.reset_view() + + def reset_view(self): + self.pack_forget() + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=20) + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + display_msg = "Reports telemetry permission required.\nPlease grant the 'Reports.Read.All' application permission to your App Registration in Entra ID." + + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 10)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + usage_logger.info("M365 App Usage trigger_fetch called. Spawning background worker thread...") + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=10) + self.grid_frame.pack_forget() + + self._set_state_loading("Downloading and parsing M365 App Usage reports...") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str): + if self.semaphore: + self.semaphore.acquire() + try: + m365_data = run_m365_pipeline(client_id, client_secret, tenant) + usage_logger.info("Successfully completed M365 Apps usage data fetch.") + self.after(0, self._render_success, m365_data) + except Exception as e: + usage_logger.error("Exception caught in M365AppUsage worker.", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, m365_data: list): + self.m365_data = m365_data + self.state_frame.pack_forget() + for w in self.grid_frame.winfo_children(): + w.destroy() + + self.grid_frame.pack(fill="x", expand=True) + + for i in range(4): + self.grid_frame.grid_columnconfigure(i, weight=1) + + headers_m365 = ["App / Platform", "Users Count", "App / Platform", "Users Count"] + for col_idx, head_text in enumerate(headers_m365): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + if not m365_data: + empty_cell = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + empty_cell.grid(row=1, column=0, columnspan=4, sticky="nsew", pady=15) + ctk.CTkLabel(empty_cell, text="No M365 App usage data found.", text_color=COLOR_TEXT_SUB).pack() + else: + half = (len(m365_data) + 1) // 2 + left_col = m365_data[:half] + right_col = m365_data[half:] + + for r_idx in range(half): + bg_style = COLOR_SURFACE if r_idx % 2 == 0 else COLOR_SURFACE_VARIANT + row_items = [] + + if r_idx < len(left_col): + row_items.extend([left_col[r_idx][0], left_col[r_idx][1]]) + else: + row_items.extend(["", ""]) + + if r_idx < len(right_col): + row_items.extend([right_col[r_idx][0], right_col[r_idx][1]]) + else: + row_items.extend(["", ""]) + + for c_idx, val in enumerate(row_items): + cell = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + cell.grid(row=r_idx + 1, column=c_idx, sticky="nsew", padx=0, pady=(0, 1)) + fnt = FONT_BODY_BOLD if c_idx in [0, 2] else FONT_BODY_MEDIUM + ctk.CTkLabel(cell, text=str(val), font=fnt, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="nw") + + self.status = "success" + self.on_status_change() + + def _render_error(self, err_msg): + usage_logger.warning(f"M365 App Usage fetch failed: {err_msg}") + self._set_state_error(err_msg) + self.status = "error" + self.on_status_change() diff --git a/telemetry/calendar_telemetry.py b/telemetry/calendar_telemetry.py new file mode 100644 index 00000000..7d04d764 --- /dev/null +++ b/telemetry/calendar_telemetry.py @@ -0,0 +1,290 @@ +import os +import logging +import threading +import customtkinter as ctk +from core.powershell.client import PowerShellClient +from telemetry.styles import * + +calendar_logger = logging.getLogger("M365TelemetryAsyncLogger.CalendarTelemetry") + +def run_calendar_telemetry_pipeline(client_id: str, client_secret: str, tenant_id: str) -> dict: + """Consolidated orchestration pipeline to download and audit Exchange Calendar telemetry config via PowerShell client.""" + from core.graph.client import GraphClient + from core.graph.directory import DirectoryService + + calendar_logger.info("Starting PowerShell Calendar Telemetry Pipeline...") + + # 1. Initialize GraphClient ONLY to get the tenant primary domain + tenant_domain = tenant_id + client = None + try: + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate() + dir_svc = DirectoryService(client) + tenant_domain = dir_svc.get_tenant_primary_domain() + calendar_logger.info(f"Retrieved primary tenant domain: {tenant_domain}") + except Exception as e: + calendar_logger.warning(f"Could not retrieve tenant domain via Graph. Falling back to Tenant ID Guid: {e}") + finally: + if client: + client.close() + + # 2. Connect to Exchange Online PowerShell for administrative metadata + rooms_count = 0 + rooms_error = None + rooms_naming = None + equipment_count = 0 + equipment_error = None + can_share_attachments = True + owa_policy_error = None + org_apps = [] + apps_error = None + powershell_error = None + + try: + calendar_logger.info("Connecting to Exchange Online PowerShell for calendar metadata...") + ps_client = PowerShellClient( + tenant_id=tenant_domain, + client_id=client_id, + client_secret=client_secret, + cert_tenant_id=tenant_id + ) + from core.powershell.calendar import CalendarStatsService + cal_service = CalendarStatsService(ps_client) + metadata = cal_service.fetch_calendar_attachments_policy() + + rooms_count = metadata.get("RoomsCount", 0) + rooms_error = metadata.get("RoomsError") + rooms_naming = metadata.get("RoomsNaming") + equipment_count = metadata.get("EquipmentCount", 0) + equipment_error = metadata.get("EquipmentError") + can_share_attachments = metadata.get("CanShareAttachments", True) + owa_policy_error = metadata.get("OwaPolicyError") + org_apps = metadata.get("OrganizationApps", []) + apps_error = metadata.get("AppsError") + + except Exception as e: + calendar_logger.warning(f"Could not connect to Exchange Online PowerShell: {e}") + powershell_error = str(e) + + if "pwsh" in str(e).lower() or "powershell" in str(e).lower(): + err_msg = "pwsh not available" + elif "module" in str(e).lower(): + err_msg = "ExchangeOnlineManagement module not installed" + else: + err_msg = "Not Permitted (Exchange Permission Issue)" + + rooms_error = err_msg + equipment_error = err_msg + owa_policy_error = err_msg + apps_error = err_msg + + # Log individual component errors + if rooms_error: + calendar_logger.error(f"Exchange PowerShell error querying Room Mailboxes: {rooms_error}") + if equipment_error: + calendar_logger.error(f"Exchange PowerShell error querying Equipment Mailboxes: {equipment_error}") + if owa_policy_error: + calendar_logger.error(f"Exchange PowerShell error querying OWA Mailbox Policy: {owa_policy_error}") + if apps_error: + calendar_logger.error(f"Exchange PowerShell error querying Organization Apps: {apps_error}") + + total_resources = rooms_count + equipment_count + + return { + "CanUsersReserveRooms": rooms_error if rooms_error else (total_resources > 0), + "TotalCalendarResources": total_resources, + "RoomsCount": rooms_count, + "EquipmentCount": equipment_count, + "RoomsError": rooms_error, + "DevicesError": equipment_error, + "OrganizationApps": org_apps, + "AppsError": apps_error, + "NamingConvention": rooms_error if rooms_error else (rooms_naming if rooms_naming else "None found"), + "CanShareAttachments": owa_policy_error if owa_policy_error else can_share_attachments, + "powershell_error": powershell_error + } + + +class CalendarTelemetryFrame(ctk.CTkFrame): + """Self-contained customtkinter component wrapping Exchange Online Calendar Environment Telemetry UI.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None + + self.build_ui() + + def build_ui(self): + """Creates card container for the tab.""" + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + ctk.CTkLabel(self.inner_pad, text="Exchange Online Calendar Environment", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(anchor="w", pady=(0, 10)) + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.warning_label = ctk.CTkLabel(self.inner_pad, text="", font=FONT_BODY_MEDIUM, text_color=COLOR_ERROR, justify="left", anchor="w", wraplength=750) + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.reset_view() + + def reset_view(self): + """Resets and hides grids.""" + self.pack_forget() + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + if hasattr(self, "warning_label"): + self.warning_label.pack_forget() + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=20) + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + ctk.CTkLabel(self.state_frame, text=f"✖ {error_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 10)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + """Triggers background fetch thread.""" + calendar_logger.info("Calendar Telemetry trigger_fetch called. Spawning background thread...") + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=10) + self.grid_frame.pack_forget() + self.warning_label.pack_forget() + + self._set_state_loading("Downloading and auditing Exchange Calendar configurations...") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str): + calendar_logger.info("Executing thread: _execute_calendar_worker") + if self.semaphore: + self.semaphore.acquire() + try: + data = run_calendar_telemetry_pipeline(client_id, client_secret, tenant) + calendar_logger.info("Successfully completed Calendar telemetry data fetch.") + self.after(0, self._render_success, data) + except Exception as e: + calendar_logger.error("Exception caught in Calendar worker.", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, data: dict): + self.last_data = data + calendar_logger.info("Calendar data successfully retrieved. Rendering UI grid.") + self.state_frame.pack_forget() + for w in self.grid_frame.winfo_children(): + w.destroy() + + if data.get("powershell_error"): + friendly_msg = f"Exchange PowerShell query failed: {data['powershell_error']}" + self.warning_label.configure(text=f"⚠️ Warning: {friendly_msg}") + self.warning_label.pack(anchor="w", pady=(0, 10)) + else: + self.warning_label.pack_forget() + + self.grid_frame.pack(fill="x", expand=True) + + self.grid_frame.grid_columnconfigure(0, weight=3) + self.grid_frame.grid_columnconfigure(1, weight=2) + + headers_sp = ["Calendar Configuration / Metric", "Value / Configuration"] + for col_idx, head_text in enumerate(headers_sp): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + def yes_no(val): + if val is None: + return "Unavailable" + return "Yes" if val else "No" + + # Formulate resources display + rooms_err = data.get("RoomsError") + devs_err = data.get("DevicesError") + rooms_count = data.get("RoomsCount", 0) + equip_count = data.get("EquipmentCount", 0) + + if rooms_err and devs_err: + res_val = rooms_err + else: + r_str = "Error" if rooms_err else str(rooms_count) + e_str = "Error" if devs_err else str(equip_count) + tot = "Error" if (rooms_err or devs_err) else str(rooms_count + equip_count) + res_val = f"Total: {tot} ({r_str} Rooms, {e_str} Equipment)" + + # Reserve rooms display + reserve_val = data.get("CanUsersReserveRooms") + if isinstance(reserve_val, bool): + reserve_val = "Yes" if reserve_val else "No" + + # Attachments display + att_val = data.get("CanShareAttachments") + if isinstance(att_val, bool): + attachments_val = "Yes" if att_val else "No" + else: + attachments_val = att_val + + rows_data = [ + ("Room & Resource Reservation", reserve_val), + ("Calendar Resources", res_val), + ("Resource Naming Convention", data.get("NamingConvention") or "None found"), + ("Calendar Attachments Enabled", attachments_val), + ] + + for r_idx, (metric_name, val) in enumerate(rows_data, start=1): + bg_style = "transparent" if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + + c0 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c0.grid(row=r_idx, column=0, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(c0, text=metric_name, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN, wraplength=450, justify="left", anchor="w").pack(padx=10, pady=6, anchor="w") + + c1 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c1.grid(row=r_idx, column=1, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(c1, text=val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, wraplength=300, justify="left", anchor="w").pack(padx=10, pady=6, anchor="w") + + self.status = "success" + self.on_status_change() + + def _render_error(self, err_msg): + calendar_logger.warning(f"Calendar Telemetry fetch failed: {err_msg}") + self._set_state_error(err_msg) + self.status = "error" + self.on_status_change() diff --git a/telemetry/data_security_governance.py b/telemetry/data_security_governance.py new file mode 100644 index 00000000..72014273 --- /dev/null +++ b/telemetry/data_security_governance.py @@ -0,0 +1,1211 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Modular Data Security and Governance telemetry scanner and visual interface.""" + +import os +import logging +import threading +import customtkinter as ctk +import webbrowser +from tkinter import messagebox, filedialog +from datetime import datetime +import pandas as pd + +from core.graph.client import GraphClient +from core.graph.security import SecurityService +from core.graph.directory import DirectoryService +from core.powershell.client import PowerShellClient +from core.powershell.retention import RetentionService + +# Bind to the async logger initialized in m365_telemetry.py +usage_logger = logging.getLogger("M365TelemetryAsyncLogger") + +# Import shared styles +from telemetry.styles import * + +def run_security_governance_pipeline(client_id, client_secret, tenant_id) -> dict: + """Pipeline specifically for security and governance policy data collection.""" + usage_logger.info("Starting Data Security & Governance Pipeline...") + + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=3, + backoff=2 + ) + client.authenticate() + service = SecurityService(client) + + labels = None + labels_error = None + + # Fetch Sensitivity Labels + try: + labels = service.fetch_sensitivity_labels() + # Sort labels by priority descending + labels.sort(key=lambda x: x.get("priority", 0), reverse=True) + except Exception as e: + usage_logger.error("Failed to fetch sensitivity labels", exc_info=True) + labels_error = str(e) + + # Fetch tenant primary domain name from organization endpoint + tenant_domain = tenant_id + try: + dir_svc = DirectoryService(client) + tenant_domain = dir_svc.get_tenant_primary_domain() + usage_logger.info(f"Retrieved primary tenant domain for Connect-IPPSSession: {tenant_domain}") + except Exception as e: + usage_logger.warning(f"Could not retrieve tenant domain via Graph. Falling back to Tenant ID Guid: {e}") + + client.close() + + # Fetch Retention Policies via PowerShell client + policies = None + policies_error = None + try: + ps_client = PowerShellClient(tenant_id=tenant_domain, client_id=client_id, client_secret=client_secret, cert_tenant_id=tenant_id) + retention_service = RetentionService(ps_client) + policies = retention_service.fetch_retention_policies() + except Exception as e: + usage_logger.error("Failed to fetch retention policies via PowerShell", exc_info=True) + policies_error = str(e) + + # Raise ConnectionError only if BOTH failed + if labels_error and policies_error: + raise ConnectionError(f"Security governance fetch failed.\nLabels Error: {labels_error}\nPolicies Error: {policies_error}") + + usage_logger.info("Data Security & Governance Pipeline completed successfully.") + return { + "labels": labels, + "labels_error": labels_error, + "policies": policies, + "policies_error": policies_error + } + +def fetch_sensitivity_labels_data(client_id, client_secret, tenant_id) -> dict: + """Fetch sensitivity labels and sort them.""" + usage_logger.info("Starting Sensitivity Labels fetch...") + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=3, + backoff=2 + ) + try: + client.authenticate() + service = SecurityService(client) + labels = service.fetch_sensitivity_labels() + # Sort labels by priority descending + labels.sort(key=lambda x: x.get("priority", 0), reverse=True) + return {"labels": labels, "error": None} + except Exception as e: + usage_logger.error("Failed to fetch sensitivity labels", exc_info=True) + return {"labels": None, "error": str(e)} + finally: + try: + client.close() + except Exception: + pass + +def fetch_retention_policies_data(client_id, client_secret, tenant_id) -> dict: + """Fetch retention policies via PowerShell client.""" + usage_logger.info("Starting Retention Policies fetch...") + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=3, + backoff=2 + ) + tenant_domain = tenant_id + try: + client.authenticate() + from core.graph.directory import DirectoryService + dir_svc = DirectoryService(client) + tenant_domain = dir_svc.get_tenant_primary_domain() + usage_logger.info(f"Retrieved primary tenant domain: {tenant_domain}") + except Exception as e: + usage_logger.warning(f"Could not retrieve tenant domain. Falling back to Tenant ID Guid: {e}") + finally: + try: + client.close() + except Exception: + pass + + try: + from core.powershell.client import PowerShellClient + from core.powershell.retention import RetentionService + + ps_client = PowerShellClient(tenant_id=tenant_domain, client_id=client_id, client_secret=client_secret, cert_tenant_id=tenant_id) + retention_service = RetentionService(ps_client) + policies = retention_service.fetch_retention_policies() + return {"policies": policies, "error": None} + except Exception as e: + usage_logger.error("Failed to fetch retention policies via PowerShell", exc_info=True) + return {"policies": None, "error": str(e)} + +def fetch_authentication_data(client_id, client_secret, tenant_id) -> dict: + """Fetch Entra ID Conditional Access authentication mechanics and Enterprise SSO modes.""" + usage_logger.info("Starting Authentication & Conditional Access fetch...") + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=2, + backoff=1 + ) + try: + client.authenticate() + service = SecurityService(client) + policies = service.fetch_conditional_access_policies() + ca_policies = [] + + for p in policies: + name = p.get("displayName", "N/A") + state = p.get("state", "N/A") + + cond = p.get("conditions") or {} + grants = p.get("grantControls") or {} + sessions = p.get("sessionControls") or {} + + users_obj = cond.get("users") or {} + users_arr = users_obj.get("includeUsers") or ["N/A"] + + apps_arr = cond.get("clientAppTypes") or ["N/A"] + + controls_arr = grants.get("builtInControls") or ["N/A"] + + session_keys = list(sessions.keys()) if sessions else ["N/A"] + + ca_policies.append({ + "name": name, + "state": state, + "users": ", ".join(users_arr), + "apps": ", ".join(apps_arr), + "controls": ", ".join(controls_arr) + }) + + return { + "auth_data": { + "ca_policies": ca_policies + }, + "error": None + } + except PermissionError as pe: + msg = str(pe) + if not msg or msg == "Policy.Read.All or Policy.Read permission required.": + msg = "Conditional Access telemetry permission required.\nPlease grant the 'Policy.Read.All' (or 'Policy.Read') application permission to your App Registration in Microsoft Entra ID." + return {"auth_data": None, "error": msg} + except Exception as e: + err_str = str(e) + if "401" in err_str or "403" in err_str or "permission" in err_str.lower() or "unauthorized" in err_str.lower(): + err_str = "Conditional Access / Application telemetry permission required.\nPlease grant the 'Policy.Read.All' and 'Application.Read.All' application permissions to your App Registration in Microsoft Entra ID." + return {"auth_data": None, "error": err_str} + finally: + try: + client.close() + except Exception: + pass + + + +class DataSecurityGovernanceFrame(ctk.CTkFrame): + """Self-contained customtkinter component wrapping Data Security & Governance UI.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None # 'loading', 'success', 'error', None + + self.current_page = 0 + self.ITEMS_PER_PAGE = 8 + self.last_labels_data = None + self.last_policies_data = None + + import tempfile + import sqlite3 + import atexit + self.db_fd, self.db_path = tempfile.mkstemp(suffix=".db") + self.conn = sqlite3.connect(self.db_path, check_same_thread=False) + self.cursor = self.conn.cursor() + self.cursor.execute('''CREATE TABLE IF NOT EXISTS labels + (id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT, description TEXT, hasProtection INTEGER, + applicationMode TEXT, priority INTEGER, + applicableTo TEXT, isEnabled INTEGER, is_sublabel INTEGER)''') + self.conn.commit() + + def cleanup_db(): + try: + self.conn.close() + import os + os.close(self.db_fd) + os.remove(self.db_path) + except Exception: + pass + atexit.register(cleanup_db) + + self.build_ui() + + def build_ui(self): + """Creates card container for the tab.""" + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + # Permanent section heading visible during loading and error states + self.main_title = ctk.CTkLabel(self.inner_pad, text="Data Security & Governance", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN) + self.main_title.pack(anchor="w", pady=(0, 10)) + + # Sensitivity Labels section header + self.labels_header_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.labels_title = ctk.CTkLabel( + self.labels_header_frame, + text="Sensitivity Labels", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN + ) + self.labels_title.pack(side="left", anchor="w") + + self.labels_link = ctk.CTkLabel( + self.labels_header_frame, + text="Open Purview Sensitivity Label Portal ↗", + font=FONT_BODY_BOLD, + text_color=COLOR_PRIMARY, + cursor="hand2" + ) + self.labels_link.pack(side="left", anchor="w", padx=(15, 0)) + self.labels_link.bind("", lambda e: webbrowser.open("https://purview.microsoft.com/informationprotection/informationprotectionlabels/sensitivitylabels")) + self.labels_link.bind("", lambda e: self.labels_link.configure(text_color=COLOR_PRIMARY_HOVER)) + self.labels_link.bind("", lambda e: self.labels_link.configure(text_color=COLOR_PRIMARY)) + + self.btn_export_labels = ctk.CTkButton( + self.labels_header_frame, + text="Export Sensitivity Labels", + font=FONT_BODY_BOLD, + fg_color="transparent", + text_color=COLOR_PRIMARY, + border_width=1, + border_color=COLOR_OUTLINE, + hover_color=COLOR_SECONDARY_HOVER, + width=180, + height=32, + corner_radius=16, + command=self.export_labels_csv, + state="disabled" + ) + self.btn_export_labels.pack(side="right", anchor="e") + + # Grid for Sensitivity Labels + self.labels_grid = ctk.CTkFrame( + self.inner_pad, + fg_color=COLOR_SURFACE, + border_color=COLOR_OUTLINE_LIGHT, + border_width=1, + corner_radius=8 + ) + + # Pagination controls frame (centered below the grid) + + # Pagination controls frame (centered below the grid) + self.pagination_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + + self.btn_prev = ctk.CTkButton( + self.pagination_frame, + text="◀ Prev", + command=self._prev_page, + width=80, + fg_color="transparent", + border_width=1, + text_color=COLOR_PRIMARY, + border_color=COLOR_PRIMARY, + hover_color=COLOR_SECONDARY_HOVER + ) + self.btn_prev.pack(side="left", padx=10) + + self.lbl_page_info = ctk.CTkLabel( + self.pagination_frame, + text="Page 1 of 1", + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_MAIN + ) + self.lbl_page_info.pack(side="left", padx=10) + + self.btn_next = ctk.CTkButton( + self.pagination_frame, + text="Next ▶", + command=self._next_page, + width=80, + fg_color="transparent", + border_width=1, + text_color=COLOR_PRIMARY, + border_color=COLOR_PRIMARY, + hover_color=COLOR_SECONDARY_HOVER + ) + self.btn_next.pack(side="left", padx=10) + + # Retention Policies section + self.retention_header_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.retention_title = ctk.CTkLabel( + self.retention_header_frame, + text="Retention Compliance Policies", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN + ) + self.retention_title.pack(side="left", anchor="w") + + self.retention_link = ctk.CTkLabel( + self.retention_header_frame, + text="Open Purview Retention Policy Portal ↗", + font=FONT_BODY_BOLD, + text_color=COLOR_PRIMARY, + cursor="hand2" + ) + self.retention_link.pack(side="left", anchor="w", padx=(15, 0)) + self.retention_link.bind("", lambda e: webbrowser.open("https://purview.microsoft.com/datalifecyclemanagement/retention")) + self.retention_link.bind("", lambda e: self.retention_link.configure(text_color=COLOR_PRIMARY_HOVER)) + self.retention_link.bind("", lambda e: self.retention_link.configure(text_color=COLOR_PRIMARY)) + + self.btn_export_retention = ctk.CTkButton( + self.retention_header_frame, + text="Export Retention Policies", + font=FONT_BODY_BOLD, + fg_color="transparent", + text_color=COLOR_PRIMARY, + border_width=1, + border_color=COLOR_OUTLINE, + hover_color=COLOR_SECONDARY_HOVER, + width=180, + height=32, + corner_radius=16, + command=self.export_retention_csv, + state="disabled" + ) + self.btn_export_retention.pack(side="right", anchor="e") + self.retention_grid = ctk.CTkFrame( + self.inner_pad, + fg_color=COLOR_SURFACE, + border_color=COLOR_OUTLINE_LIGHT, + border_width=1, + corner_radius=8 + ) + + # eDiscovery Cases section (Instructional Guidance) + + # eDiscovery Cases section (Instructional Guidance) + self.ediscovery_header_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.ediscovery_title = ctk.CTkLabel( + self.ediscovery_header_frame, + text="eDiscovery Cases", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN + ) + self.ediscovery_title.pack(side="left", anchor="w") + + self.ediscovery_body_frame = ctk.CTkFrame( + self.inner_pad, + fg_color=COLOR_SURFACE, + border_color=COLOR_OUTLINE_LIGHT, + border_width=1, + corner_radius=8 + ) + + self.ediscovery_content = ctk.CTkFrame(self.ediscovery_body_frame, fg_color="transparent") + self.ediscovery_content.pack(fill="x", padx=20, pady=20) + + lbl_inst1 = ctk.CTkLabel( + self.ediscovery_content, + text="eDiscovery cases cannot be scanned directly under standard Application permissions. To view your active cases, please navigate to Microsoft Purview:", + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_MAIN, + justify="left", + wraplength=700 + ) + lbl_inst1.pack(anchor="w", pady=(0, 8)) + + lbl_cases_link = ctk.CTkLabel( + self.ediscovery_content, + text="🔗 Open Purview eDiscovery Cases Portal", + font=FONT_BODY_BOLD, + text_color=COLOR_PRIMARY, + cursor="hand2" + ) + lbl_cases_link.pack(anchor="w", pady=(0, 15)) + lbl_cases_link.bind("", lambda e: webbrowser.open("https://purview.microsoft.com/ediscovery/casespage")) + lbl_cases_link.bind("", lambda e: lbl_cases_link.configure(text_color=COLOR_PRIMARY_HOVER)) + lbl_cases_link.bind("", lambda e: lbl_cases_link.configure(text_color=COLOR_PRIMARY)) + + lbl_inst2 = ctk.CTkLabel( + self.ediscovery_content, + text="Note: Accessing eDiscovery cases requires your administrator account to have the eDiscovery Manager role assigned in the tenant permissions page:", + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_SUB, + justify="left", + wraplength=700 + ) + lbl_inst2.pack(anchor="w", pady=(0, 8)) + + lbl_roles_link = ctk.CTkLabel( + self.ediscovery_content, + text="🔗 Assign eDiscovery Manager Role in Purview Settings", + font=FONT_BODY_BOLD, + text_color=COLOR_PRIMARY, + cursor="hand2" + ) + lbl_roles_link.pack(anchor="w") + lbl_roles_link.bind("", lambda e: webbrowser.open("https://purview.microsoft.com/settings/purviewpermissions")) + lbl_roles_link.bind("", lambda e: lbl_roles_link.configure(text_color=COLOR_PRIMARY_HOVER)) + lbl_roles_link.bind("", lambda e: lbl_roles_link.configure(text_color=COLOR_PRIMARY)) + + # Authentication Mechanics Section + self.auth_header_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.auth_title = ctk.CTkLabel( + self.auth_header_frame, + text="Authentication Mechanics (Conditional Access)", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN + ) + self.auth_title.pack(side="left", anchor="w") + + self.auth_link = ctk.CTkLabel( + self.auth_header_frame, + text="Open Microsoft Entra Conditional Access Portal ↗", + font=FONT_BODY_BOLD, + text_color=COLOR_PRIMARY, + cursor="hand2" + ) + self.auth_link.pack(side="left", anchor="w", padx=(15, 0)) + self.auth_link.bind("", lambda e: webbrowser.open("https://portal.azure.com/#view/Microsoft_AAD_IAM/ConditionalAccessBlade/~/Policies")) + self.auth_link.bind("", lambda e: self.auth_link.configure(text_color=COLOR_PRIMARY_HOVER)) + self.auth_link.bind("", lambda e: self.auth_link.configure(text_color=COLOR_PRIMARY)) + + self.auth_grid = ctk.CTkFrame( + self.inner_pad, + fg_color=COLOR_SURFACE, + border_color=COLOR_OUTLINE_LIGHT, + border_width=1, + corner_radius=8 + ) + + self.reset_view() + + def reset_view(self): + """Resets and hides grids.""" + self.pack_forget() + self.labels_header_frame.pack_forget() + self.labels_grid.pack_forget() + self.pagination_frame.pack_forget() + + self.retention_header_frame.pack_forget() + self.retention_grid.pack_forget() + + self.ediscovery_header_frame.pack_forget() + self.ediscovery_body_frame.pack_forget() + + self.auth_header_frame.pack_forget() + self.auth_grid.pack_forget() + + + for w in self.labels_grid.winfo_children(): + w.destroy() + for w in self.retention_grid.winfo_children(): + w.destroy() + for w in self.auth_grid.winfo_children(): + w.destroy() + + + self.current_page = 0 + self.last_labels_data = None + self.last_policies_data = None + + self.btn_export_labels.configure(state="disabled") + self.btn_export_retention.configure(state="disabled") + + def _set_labels_loading(self, msg="Loading..."): + for w in self.labels_grid.winfo_children(): + w.destroy() + self.labels_state_frame = ctk.CTkFrame(self.labels_grid, fg_color="transparent") + ctk.CTkLabel(self.labels_state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=20) + self.labels_state_frame.pack(fill="x", expand=True) + + def _set_labels_error(self, error_msg): + for w in self.labels_grid.winfo_children(): + w.destroy() + self.labels_state_frame = ctk.CTkFrame(self.labels_grid, fg_color="transparent") + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + display_msg = "Information Protection permission required.\nPlease grant the 'SensitivityLabels.Read.All' application permission to your App Registration in Microsoft Entra ID." + ctk.CTkLabel(self.labels_state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=20) + self.labels_state_frame.pack(fill="x", expand=True) + + def _set_retention_loading(self, msg="Loading..."): + for w in self.retention_grid.winfo_children(): + w.destroy() + self.retention_state_frame = ctk.CTkFrame(self.retention_grid, fg_color="transparent") + ctk.CTkLabel(self.retention_state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=20) + self.retention_state_frame.pack(fill="x", expand=True) + + def _set_retention_error(self, error_msg): + for w in self.retention_grid.winfo_children(): + w.destroy() + self.retention_state_frame = ctk.CTkFrame(self.retention_grid, fg_color="transparent") + display_msg = error_msg + if "is not installed or not in PATH" in error_msg.lower() or "pwsh" in error_msg.lower(): + display_msg = "PowerShell Core ('pwsh') is not installed or configured on this machine." + elif "exchangeonlinemanagement" in error_msg.lower(): + display_msg = "ExchangeOnlineManagement PowerShell module is missing." + ctk.CTkLabel(self.retention_state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=20) + self.retention_state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + """Triggers parallel fetches inside isolated background threads.""" + usage_logger.info("Data Security & Governance trigger_fetch called. Spawning background worker threads...") + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=(20, 10)) + + # Pack Sensitivity Labels Section + self.labels_header_frame.pack(fill="x", pady=(0, 10)) + self.labels_grid.pack(fill="x", expand=True, pady=(0, 15)) + self.pagination_frame.pack_forget() + self._set_labels_loading("Retrieving Sensitivity labels...") + + # Pack Retention Policies Section + self.retention_header_frame.pack(fill="x", pady=(20, 10)) + self.retention_grid.pack(fill="x", expand=True, pady=(0, 15)) + self._set_retention_loading("Retrieving Retention policies...") + + # Pack Authentication Section + self.auth_header_frame.pack(fill="x", pady=(20, 10)) + self.auth_grid.pack(fill="x", expand=True, pady=(0, 15)) + self._set_auth_loading("Retrieving Conditional Access authentication mechanics...") + + + # Pack eDiscovery Cases Section (static, show immediately) + self.ediscovery_header_frame.pack(fill="x", pady=(20, 10)) + self.ediscovery_body_frame.pack(fill="x", expand=True, pady=(0, 15)) + + self.btn_export_labels.configure(state="disabled") + self.btn_export_retention.configure(state="disabled") + + self.labels_status = "loading" + self.retention_status = "loading" + self.auth_status = "loading" + + threading.Thread( + target=self._execute_labels_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + threading.Thread( + target=self._execute_retention_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + threading.Thread( + target=self._execute_auth_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + + def _execute_labels_worker(self, tenant: str, client_id: str, client_secret: str): + usage_logger.info("Executing thread: _execute_labels_worker") + if self.semaphore: + self.semaphore.acquire() + try: + res = fetch_sensitivity_labels_data(client_id, client_secret, tenant) + self.after(0, self._handle_labels_result, res) + finally: + if self.semaphore: + self.semaphore.release() + + def _execute_retention_worker(self, tenant: str, client_id: str, client_secret: str): + usage_logger.info("Executing thread: _execute_retention_worker") + if self.semaphore: + self.semaphore.acquire() + try: + res = fetch_retention_policies_data(client_id, client_secret, tenant) + self.after(0, self._handle_retention_result, res) + finally: + if self.semaphore: + self.semaphore.release() + + def _execute_auth_worker(self, tenant: str, client_id: str, client_secret: str): + usage_logger.info("Executing thread: _execute_auth_worker") + if self.semaphore: + self.semaphore.acquire() + try: + res = fetch_authentication_data(client_id, client_secret, tenant) + self.after(0, self._handle_auth_result, res) + finally: + if self.semaphore: + self.semaphore.release() + + + def _set_auth_loading(self, msg="Loading..."): + for w in self.auth_grid.winfo_children(): + w.destroy() + self.auth_state_frame = ctk.CTkFrame(self.auth_grid, fg_color="transparent") + ctk.CTkLabel(self.auth_state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=20) + self.auth_state_frame.pack(fill="x", expand=True) + + def _set_auth_error(self, error_msg): + for w in self.auth_grid.winfo_children(): + w.destroy() + self.auth_state_frame = ctk.CTkFrame(self.auth_grid, fg_color="transparent") + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "permission" in error_msg.lower() or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower() or "policy.read" in error_msg.lower(): + display_msg = "Conditional Access telemetry permission required.\nPlease grant the 'Policy.Read.All' (or 'Policy.Read') application permission to your App Registration in Microsoft Entra ID." + ctk.CTkLabel(self.auth_state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 10)) + ctk.CTkButton(self.auth_state_frame, text="Try Again", command=self._retry_auth_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.auth_state_frame.pack(fill="x", expand=True) + + def _retry_auth_fetch(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.auth_status = "loading" + self.auth_grid.pack(fill="x", expand=True, pady=(0, 15)) + self._set_auth_loading("Retrieving Conditional Access authentication mechanics...") + threading.Thread(target=self._execute_auth_worker, args=(tenant, clients[0], secrets[0]), daemon=True).start() + if self.semaphore: + self.semaphore.release() + + def _handle_labels_result(self, result: dict): + for w in self.labels_grid.winfo_children(): + w.destroy() + + labels = result.get("labels") + err = result.get("error") + self.last_labels_data = labels + + if err: + self.labels_status = "error" + self._set_labels_error(err) + self.btn_export_labels.configure(state="disabled") + else: + self.labels_status = "success" + + if not labels: + ctk.CTkLabel(self.labels_grid, text="No Sensitivity Labels configured in this tenant.", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB).pack(padx=20, pady=20) + self.pagination_frame.pack_forget() + self.btn_export_labels.configure(state="disabled") + else: + self.btn_export_labels.configure(state="normal") + # Define column weights for proper proportional spacing + self.labels_grid.grid_columnconfigure(0, weight=2) # Label Name + self.labels_grid.grid_columnconfigure(1, weight=3) # Description + self.labels_grid.grid_columnconfigure(2, weight=1) # Protection + self.labels_grid.grid_columnconfigure(3, weight=1) # Mode + self.labels_grid.grid_columnconfigure(4, weight=1) # Priority + self.labels_grid.grid_columnconfigure(5, weight=2) # Applicable To + self.labels_grid.grid_columnconfigure(6, weight=1) # Status + + headers = ["Sensitivity Label", "Description", "Protection", "Mode", "Priority", "Applicable Targets", "Status"] + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(self.labels_grid, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + import sqlite3 + self.cursor.execute("DELETE FROM labels") + for parent in labels: + self.cursor.execute("INSERT INTO labels (name, description, hasProtection, applicationMode, priority, applicableTo, isEnabled, is_sublabel) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + (parent.get("name", "N/A"), + parent.get("description", "") or parent.get("toolTip", "") or "N/A", + 1 if parent.get("hasProtection", False) else 0, + parent.get("applicationMode", "N/A") or "N/A", + parent.get("priority", 0), + parent.get("applicableTo", ""), + 1 if parent.get("isEnabled", True) else 0, + 0)) + sublabels = parent.get("sublabels", []) + if sublabels: + sublabels_sorted = sorted(sublabels, key=lambda x: x.get("priority", 0), reverse=True) + for sub in sublabels_sorted: + self.cursor.execute("INSERT INTO labels (name, description, hasProtection, applicationMode, priority, applicableTo, isEnabled, is_sublabel) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + (f" ↳ {sub.get('name', 'N/A')}", + sub.get("description", "") or sub.get("toolTip", "") or "N/A", + 1 if sub.get("hasProtection", False) else 0, + sub.get("applicationMode", "N/A") or "N/A", + sub.get("priority", 0), + sub.get("applicableTo", ""), + 1 if sub.get("isEnabled", True) else 0, + 1)) + self.conn.commit() + self.current_page = 0 + self._display_current_page() + + self.cursor.execute("SELECT COUNT(*) FROM labels") + total_items = self.cursor.fetchone()[0] + if total_items > self.ITEMS_PER_PAGE: + self.pagination_frame.pack(pady=(5, 10)) + else: + self.pagination_frame.pack_forget() + + self._check_overall_status() + + def _handle_retention_result(self, result: dict): + for w in self.retention_grid.winfo_children(): + w.destroy() + + policies = result.get("policies") + err = result.get("error") + self.last_policies_data = policies + + if err: + self.retention_status = "error" + self._set_retention_error(err) + self.btn_export_retention.configure(state="disabled") + else: + self.retention_status = "success" + self._render_retention_policies(policies, None) + + self._check_overall_status() + + def _handle_auth_result(self, result: dict): + for w in self.auth_grid.winfo_children(): + w.destroy() + + auth_data = result.get("auth_data") + err = result.get("error") + + if err: + self.auth_status = "error" + self._set_auth_error(err) + else: + self.auth_status = "success" + self._render_authentication_card(auth_data) + + self._check_overall_status() + + def _render_authentication_card(self, auth_data: dict): + self.auth_grid.configure(fg_color=COLOR_SURFACE, border_width=1, border_color=COLOR_OUTLINE_LIGHT, corner_radius=8) + + ca_policies = auth_data.get("ca_policies", []) + + headers = ["Policy Name", "State", "Target Users", "Target Apps", "Enforced Controls"] + for i in range(5): + self.auth_grid.grid_columnconfigure(i, weight=1) + + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(self.auth_grid, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + if not ca_policies: + c0 = ctk.CTkFrame(self.auth_grid, fg_color=COLOR_SURFACE, corner_radius=0) + c0.grid(row=1, column=0, columnspan=5, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c0, text="N/A (No Conditional Access Policies configured)", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=12, anchor="w") + else: + for r_idx, policy in enumerate(ca_policies, start=1): + bg_style = COLOR_SURFACE if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + + vals = [ + policy.get("name", "N/A"), + policy.get("state", "N/A"), + policy.get("users", "N/A"), + policy.get("apps", "N/A"), + policy.get("controls", "N/A") + ] + + for c_idx, val in enumerate(vals): + c = ctk.CTkFrame(self.auth_grid, fg_color=bg_style, corner_radius=0) + c.grid(row=r_idx, column=c_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c, text=val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left", wraplength=180).pack(padx=10, pady=12, anchor="nw") + + + + + + def _check_overall_status(self): + if self.labels_status == "loading" or self.retention_status == "loading" or getattr(self, "auth_status", None) == "loading": + self.status = "loading" + elif self.labels_status == "error" and self.retention_status == "error" and getattr(self, "auth_status", None) == "error": + self.status = "error" + else: + self.status = "success" + self.on_status_change() + + def _display_current_page(self): + # Destroy existing data rows (row > 0) + for w in self.labels_grid.winfo_children(): + info = w.grid_info() + if "row" in info and int(info["row"]) > 0: + w.destroy() + + usage_logger.info(f"Displaying page {self.current_page + 1} of Sensitivity Labels.") + + self.cursor.execute("SELECT COUNT(*) FROM labels") + total_items = self.cursor.fetchone()[0] + total_pages = (total_items + self.ITEMS_PER_PAGE - 1) // self.ITEMS_PER_PAGE + if total_pages < 1: + total_pages = 1 + + # Bounds safety check + if self.current_page >= total_pages: + self.current_page = total_pages - 1 + if self.current_page < 0: + self.current_page = 0 + + start_idx = self.current_page * self.ITEMS_PER_PAGE + self.cursor.execute("SELECT name, description, hasProtection, applicationMode, priority, applicableTo, isEnabled, is_sublabel FROM labels ORDER BY id LIMIT ? OFFSET ?", (self.ITEMS_PER_PAGE, start_idx)) + page_items = self.cursor.fetchall() + + for offset, row_item in enumerate(page_items, start=1): + r_idx = offset + bg_style = COLOR_SURFACE if r_idx % 2 == 0 else COLOR_SURFACE_VARIANT + + name = row_item[0] + desc = row_item[1] + protection = "🛡️ Yes" if row_item[2] else "🔓 No" + mode = str(row_item[3]).capitalize() + priority = str(row_item[4]) + applicable = ", ".join([x.capitalize() for x in row_item[5].split(",") if x.strip()]) or "N/A" + status = "🟢 Enabled" if row_item[6] else "🔴 Disabled" + is_sublabel = bool(row_item[7]) + + name_color = COLOR_TEXT_MAIN if not is_sublabel else COLOR_TEXT_SUB + name_font = FONT_BODY_BOLD if not is_sublabel else FONT_BODY_MEDIUM + + c0 = ctk.CTkFrame(self.labels_grid, fg_color=bg_style, corner_radius=0) + c0.grid(row=r_idx, column=0, sticky="nsew", padx=0, pady=(0, 1)) + lbl_name = ctk.CTkLabel(c0, text=name, font=name_font, text_color=name_color) + lbl_name.pack(padx=10, pady=6, anchor="w") + c0.bind("", lambda e, l=lbl_name: l.configure(wraplength=e.width - 20)) + + c1 = ctk.CTkFrame(self.labels_grid, fg_color=bg_style, corner_radius=0) + c1.grid(row=r_idx, column=1, sticky="nsew", padx=0, pady=(0, 1)) + lbl_desc = ctk.CTkLabel(c1, text=desc, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN) + lbl_desc.pack(padx=10, pady=6, anchor="w") + c1.bind("", lambda e, l=lbl_desc: l.configure(wraplength=e.width - 20)) + + c2 = ctk.CTkFrame(self.labels_grid, fg_color=bg_style, corner_radius=0) + c2.grid(row=r_idx, column=2, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c2, text=protection, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + c3 = ctk.CTkFrame(self.labels_grid, fg_color=bg_style, corner_radius=0) + c3.grid(row=r_idx, column=3, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c3, text=mode, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + c4 = ctk.CTkFrame(self.labels_grid, fg_color=bg_style, corner_radius=0) + c4.grid(row=r_idx, column=4, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c4, text=priority, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + c5 = ctk.CTkFrame(self.labels_grid, fg_color=bg_style, corner_radius=0) + c5.grid(row=r_idx, column=5, sticky="nsew", padx=0, pady=(0, 1)) + lbl_app = ctk.CTkLabel(c5, text=applicable, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN) + lbl_app.pack(padx=10, pady=6, anchor="w") + c5.bind("", lambda e, l=lbl_app: l.configure(wraplength=e.width - 20)) + + c6 = ctk.CTkFrame(self.labels_grid, fg_color=bg_style, corner_radius=0) + c6.grid(row=r_idx, column=6, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c6, text=status, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + # Update page info label + self.lbl_page_info.configure(text=f"Page {self.current_page + 1} of {total_pages}") + + # Update navigation button states + if self.current_page <= 0: + self.btn_prev.configure(state="disabled", text_color=COLOR_TEXT_SUB, border_color=COLOR_OUTLINE_LIGHT) + else: + self.btn_prev.configure(state="normal", text_color=COLOR_PRIMARY, border_color=COLOR_PRIMARY) + + if self.current_page >= total_pages - 1: + self.btn_next.configure(state="disabled", text_color=COLOR_TEXT_SUB, border_color=COLOR_OUTLINE_LIGHT) + else: + self.btn_next.configure(state="normal", text_color=COLOR_PRIMARY, border_color=COLOR_PRIMARY) + + def _prev_page(self): + if self.current_page > 0: + self.current_page -= 1 + self._display_current_page() + + def _next_page(self): + self.cursor.execute("SELECT COUNT(*) FROM labels") + total_items = self.cursor.fetchone()[0] + total_pages = (total_items + self.ITEMS_PER_PAGE - 1) // self.ITEMS_PER_PAGE + if self.current_page < total_pages - 1: + self.current_page += 1 + self._display_current_page() + + def _render_error(self, err_msg): + usage_logger.warning(f"Data Security & Governance fetch failed: {err_msg}") + self._set_state_error(err_msg) + self.status = "error" + self.on_status_change() + self.btn_export_labels.configure(state="disabled") + self.btn_export_retention.configure(state="disabled") + + def _render_retention_policies(self, policies, policies_error): + if policies_error: + msg = policies_error + # Provide helpful, friendly advice if pwsh or dependency issue + if "powershell" in policies_error.lower() or "pwsh" in policies_error.lower(): + msg = "PowerShell Core ('pwsh') is not installed or configured on this machine.\nPlease refer to the Prerequisites in the README to configure it." + elif "exchangeonlinemanagement" in policies_error.lower(): + msg = "ExchangeOnlineManagement PowerShell module is missing.\nPlease run: Install-Module -Name ExchangeOnlineManagement -Scope CurrentUser" + + ctk.CTkLabel( + self.retention_grid, + text=f"✖ {msg}", + font=FONT_BODY_MEDIUM, + text_color=COLOR_ERROR, + justify="center" + ).pack(padx=20, pady=20) + self.btn_export_retention.configure(state="disabled") + elif policies is None or not policies: + ctk.CTkLabel( + self.retention_grid, + text="No Retention Compliance Policies found in this tenant.", + font=FONT_BODY_MEDIUM, + text_color=COLOR_TEXT_SUB + ).pack(padx=20, pady=20) + self.btn_export_retention.configure(state="disabled") + else: + self.btn_export_retention.configure(state="normal") + # Configure grid columns + self.retention_grid.grid_columnconfigure(0, weight=3) # Policy Name + self.retention_grid.grid_columnconfigure(1, weight=3) # Workloads + self.retention_grid.grid_columnconfigure(2, weight=2) # Duration & Trigger + self.retention_grid.grid_columnconfigure(3, weight=1) # Distribution + self.retention_grid.grid_columnconfigure(4, weight=1) # Status + + headers = ["Policy Name", "Workloads", "Duration", "Distribution", "Status"] + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(self.retention_grid, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + # Handle case where policies is a single dict rather than a list + policies_list = policies if isinstance(policies, list) else [policies] + + for r_idx, policy in enumerate(policies_list, start=1): + bg_style = "transparent" if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + + name = policy.get("Name", "N/A") + comment = policy.get("Comment", "") + workload = policy.get("Workload", "N/A") + duration_val = str(policy.get("Duration", "N/A")) + trigger_val = policy.get("RetentionTrigger", "N/A") + mode = policy.get("Mode", "Enforce") + dist_status = policy.get("DistributionStatus", "Success") + + # Format Duration nicely + duration_str = duration_val + if duration_val.lower() == "unlimited": + duration_str = "Keep Forever" + elif duration_val.isdigit(): + days = int(duration_val) + if days >= 365: + years = days / 365.0 + if years.is_integer(): + duration_str = f"{int(years)} Years ({days} days)" + else: + duration_str = f"{years:.1f} Years ({days} days)" + else: + duration_str = f"{days} days" + + # Append trigger details to duration string if present and not N/A + if trigger_val and trigger_val != "N/A": + trigger_map = { + "DateCreated": "created date", + "DateModified": "last modified date", + "DateLabeled": "labeled date" + } + friendly_trigger = trigger_map.get(trigger_val, trigger_val) + duration_str += f"\n(from {friendly_trigger})" + + # Enabled can be boolean or string + enabled_val = policy.get("Enabled", True) + if isinstance(enabled_val, str): + is_enabled = enabled_val.lower() == "true" + else: + is_enabled = bool(enabled_val) + + status = "🟢 Enabled" if is_enabled else "🔴 Disabled" + + c0 = ctk.CTkFrame(self.retention_grid, fg_color=bg_style, corner_radius=0) + c0.grid(row=r_idx, column=0, sticky="nsew", padx=1, pady=1) + + has_comment = bool(comment and comment != name) + lbl_name = ctk.CTkLabel(c0, text=name, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN) + lbl_name.pack(padx=10, pady=(6, 2) if has_comment else 6, anchor="w") + + if has_comment: + lbl_comment = ctk.CTkLabel(c0, text=comment, font=FONT_BODY_SMALL, text_color=COLOR_TEXT_SUB) + lbl_comment.pack(padx=10, pady=(0, 6), anchor="w") + c0.bind("", lambda e, l1=lbl_name, l2=lbl_comment: (l1.configure(wraplength=e.width - 20), l2.configure(wraplength=e.width - 20))) + else: + c0.bind("", lambda e, l=lbl_name: l.configure(wraplength=e.width - 20)) + + c1 = ctk.CTkFrame(self.retention_grid, fg_color=bg_style, corner_radius=0) + c1.grid(row=r_idx, column=1, sticky="nsew", padx=1, pady=1) + lbl_workload = ctk.CTkLabel(c1, text=workload, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN) + lbl_workload.pack(padx=10, pady=6, anchor="w") + c1.bind("", lambda e, l=lbl_workload: l.configure(wraplength=e.width - 20)) + + c2 = ctk.CTkFrame(self.retention_grid, fg_color=bg_style, corner_radius=0) + c2.grid(row=r_idx, column=2, sticky="nsew", padx=1, pady=1) + lbl_duration = ctk.CTkLabel(c2, text=duration_str, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left") + lbl_duration.pack(padx=10, pady=6, anchor="w") + c2.bind("", lambda e, l=lbl_duration: l.configure(wraplength=e.width - 20)) + + c3 = ctk.CTkFrame(self.retention_grid, fg_color=bg_style, corner_radius=0) + c3.grid(row=r_idx, column=3, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(c3, text=dist_status, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + c4 = ctk.CTkFrame(self.retention_grid, fg_color=bg_style, corner_radius=0) + c4.grid(row=r_idx, column=4, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(c4, text=status, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + def export_labels_csv(self): + """Prompts the user to save sensitivity labels as a detailed CSV file.""" + if not hasattr(self, "last_labels_data") or not self.last_labels_data: + messagebox.showinfo("No Data", "There is no sensitivity labels data to export. Please run a scan first.", parent=self) + return + + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + f = filedialog.asksaveasfilename( + initialfile=f"sensitivity_labels_{ts}.csv", + defaultextension=".csv", + filetypes=[("CSV Files", "*.csv"), ("All Files", "*.*")], + parent=self + ) + if not f: + return + + # Flatten the labels (including sublabels) to export detailed records + rows = [] + for parent in self.last_labels_data: + parent_id = parent.get("id", "N/A") + parent_name = parent.get("name", "N/A") + + rows.append({ + "Label ID": parent_id, + "Label Name": parent_name, + "Display Name": parent.get("displayName", "N/A"), + "Description": parent.get("description", "") or parent.get("toolTip", "") or "N/A", + "Priority": parent.get("priority", 0), + "Applicable Targets": parent.get("applicableTo", "N/A"), + "Is Enabled": parent.get("isEnabled", True), + "Is Sublabel": False, + "Parent Label ID": "", + "Parent Label Name": "" + }) + + for sub in parent.get("sublabels", []): + rows.append({ + "Label ID": sub.get("id", "N/A"), + "Label Name": sub.get("name", "N/A"), + "Display Name": sub.get("displayName", "N/A"), + "Description": sub.get("description", "") or sub.get("toolTip", "") or "N/A", + "Priority": sub.get("priority", 0), + "Applicable Targets": sub.get("applicableTo", "N/A"), + "Is Enabled": sub.get("isEnabled", True), + "Is Sublabel": True, + "Parent Label ID": parent_id, + "Parent Label Name": parent_name + }) + + try: + chunk_size = 1000 + for i in range(0, len(rows), chunk_size): + chunk = rows[i:i + chunk_size] + df = pd.DataFrame(chunk) + df.to_csv(f, mode='a' if i > 0 else 'w', header=(i == 0), index=False) + messagebox.showinfo("Export Successful", f"Sensitivity labels exported successfully to:\n{f}", parent=self) + except Exception as e: + messagebox.showerror("Export Failed", f"Failed to export CSV: {e}", parent=self) + + def export_retention_csv(self): + """Prompts the user to save retention policies as a detailed CSV file.""" + if not hasattr(self, "last_policies_data") or not self.last_policies_data: + messagebox.showinfo("No Data", "There is no retention policies data to export. Please run a scan first.", parent=self) + return + + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + f = filedialog.asksaveasfilename( + initialfile=f"retention_policies_{ts}.csv", + defaultextension=".csv", + filetypes=[("CSV Files", "*.csv"), ("All Files", "*.*")], + parent=self + ) + if not f: + return + + policies_list = self.last_policies_data if isinstance(self.last_policies_data, list) else [self.last_policies_data] + + rows = [] + for policy in policies_list: + duration_val = str(policy.get("Duration", "N/A")) + duration_str = duration_val + if duration_val.lower() == "unlimited": + duration_str = "Keep Forever" + elif duration_val.isdigit(): + days = int(duration_val) + if days >= 365: + years = days / 365.0 + duration_str = f"{int(years)} Years ({days} days)" if years.is_integer() else f"{years:.1f} Years ({days} days)" + else: + duration_str = f"{days} days" + + rows.append({ + "Policy Name": policy.get("Name", "N/A"), + "Identity": policy.get("Identity", "N/A"), + "Description / Comment": policy.get("Comment", "N/A"), + "Workloads": policy.get("Workload", "N/A"), + "Mode": policy.get("Mode", "N/A"), + "Distribution Status": policy.get("DistributionStatus", "N/A"), + "Is Enabled": policy.get("Enabled", True), + "Duration Days": duration_val, + "Duration Description": duration_str, + "Retention Action": policy.get("RetentionAction", "N/A"), + "Retention Trigger Basis": policy.get("RetentionTrigger", "N/A"), + "When Created": policy.get("WhenCreated", "N/A"), + "When Changed": policy.get("WhenChanged", "N/A"), + "Created By": policy.get("CreatedBy", "N/A"), + "Last Modified By": policy.get("LastModifiedBy", "N/A") + }) + + try: + chunk_size = 1000 + for i in range(0, len(rows), chunk_size): + chunk = rows[i:i + chunk_size] + df = pd.DataFrame(chunk) + df.to_csv(f, mode='a' if i > 0 else 'w', header=(i == 0), index=False) + messagebox.showinfo("Export Successful", f"Retention policies exported successfully to:\n{f}", parent=self) + except Exception as e: + messagebox.showerror("Export Failed", f"Failed to export CSV: {e}", parent=self) + + diff --git a/telemetry/directory.py b/telemetry/directory.py new file mode 100644 index 00000000..0b69f00b --- /dev/null +++ b/telemetry/directory.py @@ -0,0 +1,326 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Modular Directory Domains, Users & Groups summary telemetry scanners and visual interfaces.""" + +import logging +import threading +from typing import Any, Dict, List, Optional +import customtkinter as ctk + +# Import unified core service layer +from core.graph.client import GraphClient +from core.graph.directory import DirectoryService + +# Bind to the async logger initialized in m365_telemetry.py +usage_logger = logging.getLogger("M365TelemetryAsyncLogger") + +# Import shared styles +from telemetry.styles import * + +class DirectoryFrame(ctk.CTkFrame): + """Self-contained customtkinter component wrapping Directory summary (e.g. Domains, Users & Groups) UI.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, retries_var=None, backoff_var=None, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.retries = retries_var + self.backoff = backoff_var + self.status = None # 'loading', 'success', 'error', None + self.last_group_counts = {} + self.last_user_counts = {} + self.last_domains = [] + + self.build_ui() + + def build_ui(self): + """Creates card container for the tab.""" + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + # Uber Title Heading + ctk.CTkLabel( + self.inner_pad, + text="Directory Summary", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN + ).pack(anchor="w", pady=(0, 5)) + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + + # Domains sub-heading & grid + self.domains_title = ctk.CTkLabel( + self.inner_pad, + text="Domains", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN + ) + self.domains_grid = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_OUTLINE_LIGHT, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + # Divider between sections + self.divider = ctk.CTkFrame(self.inner_pad, height=1, fg_color=COLOR_OUTLINE_LIGHT) + + # Groups & Users sub-heading & grid + self.groups_users_title = ctk.CTkLabel( + self.inner_pad, + text="Groups & Users", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN + ) + self.groups_users_grid = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_OUTLINE_LIGHT, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.reset_view() + + def reset_view(self): + """Resets and hides grids.""" + self.pack_forget() + self.state_frame.pack_forget() + self.domains_title.pack_forget() + self.domains_grid.pack_forget() + self.divider.pack_forget() + self.groups_users_title.pack_forget() + self.groups_users_grid.pack_forget() + self.last_group_counts = {} + self.last_user_counts = {} + self.last_domains = [] + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.domains_grid.winfo_children(): + w.destroy() + for w in self.groups_users_grid.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=20) + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + display_msg = "Directory read permissions required.\nPlease grant the 'Directory.Read.All' permission to your App Registration in Entra ID." + + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 10)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + """Triggers Directory Telemetry fetch inside background thread.""" + usage_logger.info("Directory telemetry trigger_fetch called. Spawning background worker thread...") + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=10) + self.domains_grid.pack_forget() + self.divider.pack_forget() + self.groups_users_grid.pack_forget() + + self._set_state_loading("Fetching directory domains, users, and group counts...") + + retries_val = self.retries.get() if self.retries else 5 + backoff_val = self.backoff.get() if self.backoff else 2 + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret, retries_val, backoff_val), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str, retries_val: int, backoff_val: int): + usage_logger.info("Executing thread: _execute_directory_worker") + if self.semaphore: + self.semaphore.acquire() + try: + self.log_msg("Authenticating app for directory query...") + + client = GraphClient( + tenant_id=tenant, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=retries_val, + backoff=backoff_val + ) + + required_scopes = ["Directory.Read.All"] + client.authenticate(required_scopes=required_scopes) + + self.log_msg("Querying directory domains, users, and group counts from Microsoft Graph...") + dir_service = DirectoryService(client) + telemetry_data = dir_service.get_directory_telemetry(self.log_msg) + client.close() + + usage_logger.info("Successfully fetched directory telemetry data.") + self.after(0, self._render_success, telemetry_data) + except Exception as e: + usage_logger.error("Exception caught in Directory worker.", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, telemetry_dict: Dict[str, Any]): + usage_logger.info("Executing UI render for Directory domains, users & groups tables.") + self.state_frame.pack_forget() + + for w in self.domains_grid.winfo_children(): + w.destroy() + for w in self.groups_users_grid.winfo_children(): + w.destroy() + + # Display UI titles and tables + self.domains_title.pack(anchor="w", pady=(10, 10)) + self.domains_grid.pack(fill="x", expand=True, pady=(0, 10)) + + self.divider.pack(fill="x", pady=15) + + self.groups_users_title.pack(anchor="w", pady=(10, 10)) + self.groups_users_grid.pack(fill="x", expand=True, pady=(0, 10)) + + # ---------------------------------------------------- + # RENDER DOMAINS GRID + # ---------------------------------------------------- + self.domains_grid.grid_columnconfigure((0, 4), weight=3) + self.domains_grid.grid_columnconfigure((1, 2, 3), weight=2) + + domains_headers = ["Domain ID", "Admin Managed", "Default", "Verified", "Supported Services"] + for col_idx, head_text in enumerate(domains_headers): + cell = ctk.CTkFrame(self.domains_grid, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + domains_list = telemetry_dict.get("domains", []) + self.last_domains = domains_list + + if not domains_list: + empty_cell = ctk.CTkFrame(self.domains_grid, fg_color="transparent") + empty_cell.grid(row=1, column=0, columnspan=5, sticky="nsew", pady=15) + ctk.CTkLabel(empty_cell, text="No domains found under the organization.", text_color=COLOR_TEXT_SUB).pack() + else: + for item_idx, domain in enumerate(domains_list, start=1): + bg_style = COLOR_SURFACE if item_idx % 2 == 0 else COLOR_SURFACE_VARIANT + + admin_managed = "Yes" if domain.get("isAdminManaged") else "No" + is_default = "Yes" if domain.get("isDefault") else "No" + is_verified = "Yes" if domain.get("isVerified") else "No" + services = domain.get("supportedServices", []) + services_str = ", ".join(services) if services else "-" + + # Domain ID + c0 = ctk.CTkFrame(self.domains_grid, fg_color=bg_style, corner_radius=0) + c0.grid(row=item_idx, column=0, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c0, text=domain.get("id", "-"), font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=8, anchor="nw") + + # Admin Managed + c1 = ctk.CTkFrame(self.domains_grid, fg_color=bg_style, corner_radius=0) + c1.grid(row=item_idx, column=1, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c1, text=admin_managed, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=8, anchor="nw") + + # Default + c2 = ctk.CTkFrame(self.domains_grid, fg_color=bg_style, corner_radius=0) + c2.grid(row=item_idx, column=2, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c2, text=is_default, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=8, anchor="nw") + + # Verified + c3 = ctk.CTkFrame(self.domains_grid, fg_color=bg_style, corner_radius=0) + c3.grid(row=item_idx, column=3, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c3, text=is_verified, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=8, anchor="nw") + + # Supported Services + c4 = ctk.CTkFrame(self.domains_grid, fg_color=bg_style, corner_radius=0) + c4.grid(row=item_idx, column=4, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c4, text=services_str, text_color=COLOR_TEXT_MAIN, justify="left", wraplength=250).pack(padx=10, pady=8, anchor="nw") + + # ---------------------------------------------------- + # RENDER GROUPS & USERS GRID + # ---------------------------------------------------- + self.groups_users_grid.grid_columnconfigure(0, weight=3) + self.groups_users_grid.grid_columnconfigure(1, weight=1) + + groups_users_headers = ["Category", "Count"] + for col_idx, head_text in enumerate(groups_users_headers): + cell = ctk.CTkFrame(self.groups_users_grid, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + counts_dict = telemetry_dict.get("group_counts", {}) + self.last_group_counts = counts_dict + + user_counts_dict = telemetry_dict.get("user_counts", {}) + self.last_user_counts = user_counts_dict + + rows_data = [ + # User statistics + ("Total Users", user_counts_dict.get("total", 0), True), + ("Enabled Users", user_counts_dict.get("enabled", 0), False), + ("Disabled Users", user_counts_dict.get("disabled", 0), False), + ("Member Users", user_counts_dict.get("member", 0), False), + ("Guest Users", user_counts_dict.get("guest", 0), False), + # Divider separator row + (None, None, False), + # Group statistics + ("Total Groups", counts_dict.get("total", 0), True), + ("Microsoft 365 Groups (Unified)", counts_dict.get("m365", 0), False), + ("Security Groups (Static, non-mail-enabled)", counts_dict.get("security", 0), False), + ("Mail-enabled Security Groups", counts_dict.get("mail_enabled_security", 0), False), + ("Distribution Groups", counts_dict.get("distribution", 0), False), + ("Dynamic Groups (Dynamic Membership)", counts_dict.get("dynamic", 0), False) + ] + + current_row = 1 + for item in rows_data: + metric_name, val, is_bold = item + if metric_name is None: + for c_idx in range(2): + c = ctk.CTkFrame(self.groups_users_grid, fg_color=COLOR_OUTLINE_LIGHT, corner_radius=0, height=2) + c.grid(row=current_row, column=c_idx, sticky="nsew", padx=0, pady=(0, 1)) + current_row += 1 + continue + + bg_style = COLOR_SURFACE if current_row % 2 == 0 else COLOR_SURFACE_VARIANT + font_style = FONT_BODY_BOLD if is_bold else FONT_BODY_MEDIUM + + c0 = ctk.CTkFrame(self.groups_users_grid, fg_color=bg_style, corner_radius=0) + c0.grid(row=current_row, column=0, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c0, text=metric_name, font=font_style, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=8, anchor="nw") + + c1 = ctk.CTkFrame(self.groups_users_grid, fg_color=bg_style, corner_radius=0) + c1.grid(row=current_row, column=1, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c1, text=f"{val:,}", font=font_style, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=8, anchor="nw") + + current_row += 1 + + self.status = "success" + self.on_status_change() + + def _render_error(self, err_msg): + usage_logger.warning(f"Rendering Directory error state: {err_msg}") + self._set_state_error(err_msg) + self.status = "error" + self.on_status_change() diff --git a/telemetry/email_client_support.py b/telemetry/email_client_support.py new file mode 100644 index 00000000..1436cffe --- /dev/null +++ b/telemetry/email_client_support.py @@ -0,0 +1,367 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Standalone Exchange Online Supported Email Clients telemetry scanner and UI presentation.""" + +import os +import logging +import threading +import pandas as pd +import customtkinter as ctk + +from core.graph.client import GraphClient +from core.graph.reports import ReportsService +from telemetry.styles import * + +usage_logger = logging.getLogger("M365TelemetryAsyncLogger") + +def parse_email_client_support_csv(filepath: str) -> dict: + """Parses the Email App Usage Counts CSV to categorize Browser vs Non-Browser client adoption.""" + usage_logger.info(f"Processing Email App Usage Counts file: {os.path.basename(filepath)}") + if not os.path.exists(filepath): + usage_logger.warning(f"Email App Usage report {filepath} not found. Skipping client breakdown.") + return {} + + try: + df = pd.read_csv(filepath) + for col in df.columns: + if col not in ["Report Refresh Date", "User Principal Name", "Display Name", "Last Activity Date", "Is Deleted"]: + df[col] = df[col].astype(str).str.strip().str.upper().isin(["TRUE", "1", "UNDETERMINED"]) + + if "Is Deleted" in df.columns: + df = df[~df["Is Deleted"].astype(str).str.strip().str.upper().isin(["TRUE", "1"])] + + if df.empty: + usage_logger.warning(f"Email App Usage report {filepath} is empty.") + return {} + + owa_users = int(df["Outlook For Web"].sum()) if "Outlook For Web" in df.columns else 0 + + win_users = int(df["Outlook For Windows"].sum()) if "Outlook For Windows" in df.columns else 0 + mac_users = int(df["Outlook For Mac"].sum()) if "Outlook For Mac" in df.columns else 0 + mail_mac = int(df["Mail For Mac"].sum()) if "Mail For Mac" in df.columns else 0 + + mobile_users = int(df["Outlook For Mobile"].sum()) if "Outlook For Mobile" in df.columns else 0 + other_mobile = int(df["Other For Mobile"].sum()) if "Other For Mobile" in df.columns else 0 + + pop3_users = int(df["POP3 App"].sum()) if "POP3 App" in df.columns else 0 + imap4_users = int(df["IMAP4 App"].sum()) if "IMAP4 App" in df.columns else 0 + smtp_users = int(df["SMTP App"].sum()) if "SMTP App" in df.columns else 0 + + total_desktop = win_users + mac_users + mail_mac + total_mobile = mobile_users + other_mobile + total_protocols = pop3_users + imap4_users + smtp_users + total_non_browser = total_desktop + total_mobile + total_protocols + + return { + "browser_users": owa_users, + "desktop_win": win_users, + "desktop_mac": mac_users, + "desktop_mail_mac": mail_mac, + "mobile_outlook": mobile_users, + "mobile_other": other_mobile, + "protocol_pop3": pop3_users, + "protocol_imap4": imap4_users, + "protocol_smtp": smtp_users, + "total_desktop": total_desktop, + "total_mobile": total_mobile, + "total_protocols": total_protocols, + "total_non_browser": total_non_browser + } + except Exception as e: + usage_logger.error(f"Error parsing Email App Usage CSV: {e}") + return {} + +def run_email_client_pipeline(client_id: str, client_secret: str, tenant_id: str) -> dict: + """Pipeline specifically downloading getEmailAppUsageAppsUserCounts reports and discovering PSTs.""" + usage_logger.info("Starting Email Environment & PST Discovery Pipeline...") + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=3, + backoff=2 + ) + client.authenticate() + service = ReportsService(client) + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + reports_dir = os.path.join(script_dir, "reports", f"{tenant_id}_{client_id}") + + client_stats = {} + client_error = None + try: + service.download_email_app_usage_detail(reports_dir) + client_stats = parse_email_client_support_csv(os.path.join(reports_dir, "EmailAppUsageUserDetail(180d).csv")) + except Exception as e: + usage_logger.warning(f"Could not retrieve/parse Email App Usage report: {e}") + client_error = str(e) + + pst_cloud = {} + pst_error = None + try: + pst_cloud = service.search_cloud_pst_files() + except Exception as e: + pst_error = str(e) + + client.close() + return { + "client_adoption": client_stats, + "client_error": client_error, + "pst_cloud_data": pst_cloud, + "pst_error": pst_error + } + + +class EmailClientSupportFrame(ctk.CTkFrame): + """Self-contained CustomTkinter component rendering Email Environment UI.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None + + self.build_ui() + + def build_ui(self): + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + self.sub_title_1 = ctk.CTkLabel(self.inner_pad, text="Email Client Classification", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN) + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.sub_title_2 = ctk.CTkLabel(self.inner_pad, text="PST Files", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN) + self.pst_grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.reset_view() + + def reset_view(self): + self.pack_forget() + self.sub_title_1.pack_forget() + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + self.sub_title_2.pack_forget() + self.pst_grid_frame.pack_forget() + if hasattr(self, "pst_disclaimer_lbl") and self.pst_disclaimer_lbl: + self.pst_disclaimer_lbl.pack_forget() + self.pst_disclaimer_lbl.destroy() + self.pst_disclaimer_lbl = None + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + for w in self.pst_grid_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self): + self.state_frame.pack_forget() + + self.sub_title_1.pack(anchor="w", pady=(5, 10)) + self.grid_frame.pack(fill="x", expand=True) + for w in self.grid_frame.winfo_children(): + w.destroy() + f1 = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + ctk.CTkLabel(f1, text="⏳ Analyzing Email Clients...", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=20) + f1.pack(fill="x", expand=True) + + self.sub_title_2.pack(anchor="w", pady=(20, 10)) + self.pst_grid_frame.pack(fill="x", expand=True) + for w in self.pst_grid_frame.winfo_children(): + w.destroy() + f2 = ctk.CTkFrame(self.pst_grid_frame, fg_color="transparent") + ctk.CTkLabel(f2, text="⏳ Discovering PST Files...", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=20) + f2.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + self.sub_title_1.pack_forget() + self.grid_frame.pack_forget() + self.sub_title_2.pack_forget() + self.pst_grid_frame.pack_forget() + + for w in self.state_frame.winfo_children(): + w.destroy() + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower(): + display_msg = "Reports / Search telemetry permission required.\nPlease grant 'Reports.Read.All' and 'Files.Read.All' in Microsoft Entra ID." + + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 10)) + self.state_frame.pack(fill="x", expand=True) + + def trigger_fetch(self, tenant, client_id, client_secret): + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=10) + self._set_state_loading() + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str): + if self.semaphore: + self.semaphore.acquire() + try: + data = run_email_client_pipeline(client_id, client_secret, tenant) + self.after(0, self._render_success, data) + except Exception as e: + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, data: dict): + self.state_frame.pack_forget() + for w in self.grid_frame.winfo_children(): + w.destroy() + for w in self.pst_grid_frame.winfo_children(): + w.destroy() + + # Render Supported Email Clients + self.sub_title_1.pack(anchor="w", pady=(5, 10)) + self.grid_frame.pack(fill="x", expand=True) + for i in range(2): + self.grid_frame.grid_columnconfigure(i, weight=1) + + headers_client = ["Email Client Classification", "Active User Counts (180-Day Telemetry)"] + for col_idx, head_text in enumerate(headers_client): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + client_err = data.get("client_error") + if client_err: + d_str = f"✖ Error: {client_err}" + m_str = f"✖ Error: {client_err}" + p_str = f"✖ Error: {client_err}" + rows_client = [ + ("Supported Browser-Based Clients", f"✖ Error: {client_err}"), + ("Supported Non-Browser (Desktop)", d_str), + ("Supported Non-Browser (Mobile)", m_str), + ("Supported Non-Browser (Protocols)", p_str) + ] + else: + c_adop = data.get("client_adoption", {}) + b_users = c_adop.get("browser_users", 0) if c_adop else 0 + d_win = c_adop.get("desktop_win", 0) if c_adop else 0 + d_mac = c_adop.get("desktop_mac", 0) if c_adop else 0 + m_mac = c_adop.get("desktop_mail_mac", 0) if c_adop else 0 + m_out = c_adop.get("mobile_outlook", 0) if c_adop else 0 + m_oth = c_adop.get("mobile_other", 0) if c_adop else 0 + p_imap = c_adop.get("protocol_imap4", 0) if c_adop else 0 + p_smtp = c_adop.get("protocol_smtp", 0) if c_adop else 0 + p_pop = c_adop.get("protocol_pop3", 0) if c_adop else 0 + + d_str = (f"• Outlook for Windows: {d_win:,} Users\n" + f"• Outlook for Mac: {d_mac:,} Users\n" + f"• Apple Mail (macOS): {m_mac:,} Users") + + m_str = (f"• Outlook Mobile (iOS/Android): {m_out:,} Users\n" + f"• Native / Other Mobile Apps: {m_oth:,} Users") + + p_str = (f"• IMAP4 App: {p_imap:,} Users\n" + f"• POP3 App: {p_pop:,} Users\n" + f"• SMTP App: {p_smtp:,} Accounts") + + rows_client = [ + ("Supported Browser-Based Clients", f"• Outlook on the Web (OWA): {b_users:,} Users"), + ("Supported Non-Browser (Desktop)", d_str), + ("Supported Non-Browser (Mobile)", m_str), + ("Supported Non-Browser (Protocols)", p_str) + ] + + for cr_idx, (c_name, c_val) in enumerate(rows_client, start=1): + bg_c = "transparent" if cr_idx % 2 != 0 else COLOR_SURFACE_VARIANT + + cc0 = ctk.CTkFrame(self.grid_frame, fg_color=bg_c, corner_radius=0) + cc0.grid(row=cr_idx, column=0, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cc0, text=c_name, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=10, anchor="nw") + + cc1 = ctk.CTkFrame(self.grid_frame, fg_color=bg_c, corner_radius=0) + cc1.grid(row=cr_idx, column=1, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cc1, text=c_val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left").pack(padx=10, pady=10, anchor="nw") + + # 3. Subsection 3: PST Files + self.sub_title_2.pack(anchor="w", pady=(25, 10)) + self.pst_grid_frame.pack(fill="x", expand=True) + self.pst_grid_frame.grid_columnconfigure(0, weight=2) + self.pst_grid_frame.grid_columnconfigure(1, weight=5) + + headers_pst = ["PST Storage Location", "Discovered File Count & Size"] + for col_idx, head_text in enumerate(headers_pst): + cell = ctk.CTkFrame(self.pst_grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + pst_err = data.get("pst_error") + if pst_err: + cloud_str = f"✖ Error: {pst_err}" + else: + pst_cloud = data.get("pst_cloud_data", {}) + cloud_count = 0 + cloud_bytes = 0 + if pst_cloud and "value" in pst_cloud: + for item in pst_cloud.get("value", []): + for hc in item.get("hitsContainers", []): + cloud_count += hc.get("total", 0) + for hit in hc.get("hits", []): + cloud_bytes += int(hit.get("resource", {}).get("size", 0)) + + cloud_size_str = f" ({format_bytes(cloud_bytes)})" if cloud_bytes > 0 else "" + cloud_str = f"{cloud_count:,} Files{cloud_size_str}" if cloud_count > 0 else "None Detected" + + rows_pst = [ + ("Cloud (SharePoint & OneDrive)", cloud_str) + ] + + for p_idx, (p_name, p_val) in enumerate(rows_pst, start=1): + bg_p = "transparent" if p_idx % 2 != 0 else COLOR_SURFACE_VARIANT + + pp0 = ctk.CTkFrame(self.pst_grid_frame, fg_color=bg_p, corner_radius=0) + pp0.grid(row=p_idx, column=0, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(pp0, text=p_name, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=10, anchor="nw") + + pp1 = ctk.CTkFrame(self.pst_grid_frame, fg_color=bg_p, corner_radius=0) + pp1.grid(row=p_idx, column=1, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(pp1, text=p_val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left").pack(padx=10, pady=10, anchor="nw") + + # Show disclaimer if any cloud PST files are found + if not pst_err and cloud_count > 0: + self.pst_disclaimer_lbl = ctk.CTkLabel( + self.inner_pad, + text="* Note: There may be more than 2,000 files in the tenant; this tool only checks up to 2,000 files.", + font=FONT_BODY_SMALL, + text_color=COLOR_TEXT_SUB, + justify="left" + ) + self.pst_disclaimer_lbl.pack(anchor="w", pady=(10, 0)) + + self.status = "success" + self.on_status_change() + + def _render_error(self, err_msg): + self._set_state_error(err_msg) + self.status = "error" + self.on_status_change() + diff --git a/telemetry/exchange_apps.py b/telemetry/exchange_apps.py new file mode 100644 index 00000000..06623311 --- /dev/null +++ b/telemetry/exchange_apps.py @@ -0,0 +1,204 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Modular Exchange Online Organization-wide Apps telemetry scanners and visual interfaces.""" + +import logging +import threading +from typing import Any, Dict, List, Optional +import customtkinter as ctk + +# Bind to the async logger initialized in m365_telemetry.py +usage_logger = logging.getLogger("M365TelemetryAsyncLogger") + +# Import shared styles +from telemetry.styles import * +from telemetry.calendar_telemetry import run_calendar_telemetry_pipeline + +class ExchangeAppsFrame(ctk.CTkFrame): + """Self-contained customtkinter component wrapping Exchange Online Org-wide Apps UI with side-by-side columns layout.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None + self.last_apps = [] + + self.build_ui() + + def build_ui(self): + """Creates card container for the tab.""" + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + ctk.CTkLabel(self.inner_pad, text="Integrated Apps", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(anchor="w", pady=(0, 10)) + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.warning_label = ctk.CTkLabel(self.inner_pad, text="", font=FONT_BODY_MEDIUM, text_color=COLOR_ERROR, justify="left", anchor="w", wraplength=750) + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.reset_view() + + def reset_view(self): + """Resets and hides grids.""" + self.pack_forget() + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + self.last_apps = [] + if hasattr(self, "warning_label"): + self.warning_label.pack_forget() + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=20) + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + ctk.CTkLabel(self.state_frame, text=f"✖ {error_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 10)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + """Triggers background fetch thread.""" + usage_logger.info("Exchange Apps trigger_fetch called. Spawning background thread...") + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=10) + self.grid_frame.pack_forget() + self.warning_label.pack_forget() + + self._set_state_loading("Downloading Exchange organization-wide apps configurations...") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str): + usage_logger.info("Executing thread: _execute_apps_worker") + if self.semaphore: + self.semaphore.acquire() + try: + data = run_calendar_telemetry_pipeline(client_id, client_secret, tenant) + usage_logger.info("Successfully completed Exchange Apps telemetry data fetch.") + self.after(0, self._render_success, data) + except Exception as e: + usage_logger.error("Exception caught in Exchange Apps worker.", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, data: dict): + usage_logger.info("Exchange Apps data successfully retrieved. Rendering UI grid.") + self.state_frame.pack_forget() + for w in self.grid_frame.winfo_children(): + w.destroy() + + if data.get("powershell_error"): + friendly_msg = f"Exchange PowerShell query failed: {data['powershell_error']}" + self.warning_label.configure(text=f"⚠️ Warning: {friendly_msg}") + self.warning_label.pack(anchor="w", pady=(0, 10)) + else: + self.warning_label.pack_forget() + + self.grid_frame.pack(fill="x", expand=True) + + for i in range(4): + self.grid_frame.grid_columnconfigure(i, weight=1) + + headers = ["App Display Name", "Status", "App Display Name", "Status"] + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + apps_list = data.get("OrganizationApps", []) + apps_err = data.get("AppsError") + self.last_apps = apps_list + + if apps_err: + usage_logger.error(f"Exchange PowerShell query failed for organization apps: {apps_err}") + empty_cell = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + empty_cell.grid(row=1, column=0, columnspan=4, sticky="nsew", pady=15) + ctk.CTkLabel(empty_cell, text=f"Error retrieving apps: {apps_err}", text_color=COLOR_ERROR).pack() + elif not apps_list: + empty_cell = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + empty_cell.grid(row=1, column=0, columnspan=4, sticky="nsew", pady=15) + ctk.CTkLabel(empty_cell, text="No organization-wide apps found.", text_color=COLOR_TEXT_SUB).pack() + else: + half = (len(apps_list) + 1) // 2 + left_col = apps_list[:half] + right_col = apps_list[half:] + + for r_idx in range(half): + bg_style = "transparent" if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + row_items = [] + + if r_idx < len(left_col): + app = left_col[r_idx] + status_str = "Enabled" if app.get("Enabled") else "Disabled" + row_items.extend([app.get("DisplayName", "-"), status_str]) + else: + row_items.extend(["", ""]) + + if r_idx < len(right_col): + app = right_col[r_idx] + status_str = "Enabled" if app.get("Enabled") else "Disabled" + row_items.extend([app.get("DisplayName", "-"), status_str]) + else: + row_items.extend(["", ""]) + + for c_idx, val in enumerate(row_items): + cell = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + cell.grid(row=r_idx + 1, column=c_idx, sticky="nsew", padx=1, pady=1) + + is_enabled_col = c_idx in [1, 3] + fnt = FONT_BODY_MEDIUM if is_enabled_col else FONT_BODY_BOLD + + text_color = COLOR_TEXT_MAIN + if is_enabled_col and val == "Disabled": + text_color = COLOR_TEXT_SUB + + ctk.CTkLabel(cell, text=val, font=fnt, text_color=text_color, wraplength=200, justify="left", anchor="w").pack(padx=10, pady=6, anchor="w") + + self.status = "success" + self.on_status_change() + + def _render_error(self, err_msg): + usage_logger.warning(f"Exchange Apps Telemetry fetch failed: {err_msg}") + self._set_state_error(err_msg) + self.status = "error" + self.on_status_change() diff --git a/telemetry/exchange_connectors_ui.py b/telemetry/exchange_connectors_ui.py new file mode 100644 index 00000000..a3612c59 --- /dev/null +++ b/telemetry/exchange_connectors_ui.py @@ -0,0 +1,258 @@ +import customtkinter as ctk +import threading +import logging +import traceback +import webbrowser +from telemetry.styles import * + +usage_logger = logging.getLogger("ExchangeConnectorsUI") + +def fetch_exchange_connectors_data(client_id, client_secret, tenant_id) -> dict: + """Fetch Exchange Online Inbound and Outbound Connectors.""" + usage_logger.info("Starting Exchange Connectors fetch...") + + tenant_domain = tenant_id + try: + from core.graph.client import GraphClient + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=3, + backoff=2 + ) + client.authenticate() + from core.graph.directory import DirectoryService + dir_svc = DirectoryService(client) + tenant_domain = dir_svc.get_tenant_primary_domain() + usage_logger.info(f"Retrieved primary tenant domain for Connectors: {tenant_domain}") + except Exception as e: + usage_logger.warning(f"Could not retrieve tenant domain. Falling back to Tenant ID Guid: {e}") + finally: + try: + client.close() + except Exception: + pass + + try: + from core.powershell.client import PowerShellClient + from core.powershell.exchange_connectors import ExchangeConnectorsService + + ps_client = PowerShellClient(tenant_id=tenant_domain, client_id=client_id, client_secret=client_secret, cert_tenant_id=tenant_id) + connector_svc = ExchangeConnectorsService(ps_client) + data = connector_svc.fetch_exchange_connectors() + return {"connectors": data, "error": None} + except Exception as e: + usage_logger.error("Failed to fetch Exchange Connectors via PowerShell", exc_info=True) + return {"connectors": None, "error": str(e)} + +class ExchangeConnectorsFrame(ctk.CTkFrame): + """Self-contained component for rendering Exchange Online Connectors routing logic matching ExchangeOnline UI standards.""" + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None + + self.build_ui() + + def build_ui(self): + """Creates card container for the tab.""" + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + self.header_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 10)) + + self.title_lbl = ctk.CTkLabel( + self.header_frame, + text="Exchange Connectors (Inbound & Outbound Routing)", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN + ) + self.title_lbl.pack(side="left", anchor="w") + + # Link label redirecting to EAC Connectors + self.link_lbl = ctk.CTkLabel( + self.header_frame, + text="Open Exchange Admin Center ↗", + font=FONT_BODY_BOLD, + text_color=COLOR_PRIMARY, + cursor="hand2" + ) + self.link_lbl.pack(side="left", anchor="w", padx=(15, 0)) + self.link_lbl.bind("", lambda e: webbrowser.open("https://admin.cloud.microsoft/exchange?#/connectors")) + self.link_lbl.bind("", lambda e: self.link_lbl.configure(text_color=COLOR_PRIMARY_HOVER)) + self.link_lbl.bind("", lambda e: self.link_lbl.configure(text_color=COLOR_PRIMARY)) + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.reset_view() + + def reset_view(self): + self.pack_forget() + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=20) + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + display_msg = error_msg + if "is not installed or not in PATH" in error_msg.lower() or "pwsh" in error_msg.lower(): + display_msg = "PowerShell Core ('pwsh') is not installed or configured on this machine." + elif "exchangeonlinemanagement" in error_msg.lower(): + display_msg = "ExchangeOnlineManagement PowerShell module is missing." + + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 10)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + usage_logger.info("Exchange Connectors trigger_fetch called. Spawning background thread...") + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=10) + self.grid_frame.pack_forget() + + self._set_state_loading("Retrieving Exchange Connectors routing configurations...") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str): + usage_logger.info("Executing thread: _execute_connectors_worker") + if self.semaphore: + self.semaphore.acquire() + try: + res = fetch_exchange_connectors_data(client_id, client_secret, tenant) + self.after(0, self._handle_result, res) + finally: + if self.semaphore: + self.semaphore.release() + + def _handle_result(self, result: dict): + self.state_frame.pack_forget() + for w in self.grid_frame.winfo_children(): + w.destroy() + + connectors_data = result.get("connectors") + err = result.get("error") + + if err: + self.status = "error" + self._set_state_error(err) + else: + ps_errs = connectors_data.get("Errors", {}) if connectors_data else {} + if ps_errs: + err_msg = "\n".join([f"{k}: {v}" for k,v in ps_errs.items()]) + self.status = "error" + self._set_state_error(f"PowerShell Execution Error:\n{err_msg}") + else: + self.status = "success" + self.grid_frame.pack(fill="x", expand=True) + self._render_card(connectors_data) + + self.on_status_change() + + def _render_card(self, connectors_data: dict): + if not connectors_data: + return + + inbound = connectors_data.get("InboundConnectors", []) + outbound = connectors_data.get("OutboundConnectors", []) + + if not inbound and not outbound: + self.grid_frame.configure(fg_color=COLOR_SURFACE, border_width=1, border_color=COLOR_OUTLINE_LIGHT, corner_radius=8) + ctk.CTkLabel(self.grid_frame, text="N/A (No Exchange Connectors configured)", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB).pack(padx=20, pady=20, anchor="w") + return + + # Inbound Connectors Section + if inbound: + in_header = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + in_header.pack(fill="x", padx=10, pady=(10, 5)) + ctk.CTkLabel(in_header, text="Inbound Routing (On-Premises / Third-Party Filter to Exchange Online)", font=FONT_BODY_BOLD, text_color=COLOR_PRIMARY).pack(side="left") + + in_grid = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_SURFACE, corner_radius=0) + in_grid.pack(fill="x", padx=10, pady=(0, 15)) + + headers_in = ["Connector Name", "Status", "Connector Type", "Sender Domains", "Require TLS"] + for i in range(5): + in_grid.grid_columnconfigure(i, weight=1) + + for col_idx, head_text in enumerate(headers_in): + cell = ctk.CTkFrame(in_grid, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + for r_idx, conn in enumerate(inbound, start=1): + bg_style = COLOR_SURFACE if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + vals = [ + conn.get("Name", "N/A"), + "🟢 Enabled" if conn.get("Enabled") else "🔴 Disabled", + conn.get("ConnectorType", "N/A"), + conn.get("SenderDomains", "N/A") or "N/A", + "Yes" if conn.get("RequireTls") else "No" + ] + for c_idx, val in enumerate(vals): + c = ctk.CTkFrame(in_grid, fg_color=bg_style, corner_radius=0) + c.grid(row=r_idx, column=c_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c, text=val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left", wraplength=180).pack(padx=10, pady=12, anchor="nw") + + # Outbound Connectors Section + if outbound: + out_header = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + out_header.pack(fill="x", padx=10, pady=(10, 5)) + ctk.CTkLabel(out_header, text="Outbound Routing (Exchange Online to On-Premises / Third-Party Gateway)", font=FONT_BODY_BOLD, text_color=COLOR_PRIMARY).pack(side="left") + + out_grid = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_SURFACE, corner_radius=0) + out_grid.pack(fill="x", padx=10, pady=(0, 10)) + + headers_out = ["Connector Name", "Status", "Recipient Domains", "Smart Hosts", "Use MX Record"] + for i in range(5): + out_grid.grid_columnconfigure(i, weight=1) + + for col_idx, head_text in enumerate(headers_out): + cell = ctk.CTkFrame(out_grid, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + for r_idx, conn in enumerate(outbound, start=1): + bg_style = COLOR_SURFACE if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + vals = [ + conn.get("Name", "N/A"), + "🟢 Enabled" if conn.get("Enabled") else "🔴 Disabled", + conn.get("RecipientDomains", "N/A") or "N/A", + conn.get("SmartHosts", "N/A") or "N/A", + "Yes" if conn.get("UseMxRecord") else "No" + ] + for c_idx, val in enumerate(vals): + c = ctk.CTkFrame(out_grid, fg_color=bg_style, corner_radius=0) + c.grid(row=r_idx, column=c_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c, text=val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left", wraplength=180).pack(padx=10, pady=12, anchor="nw") diff --git a/telemetry/exchange_online.py b/telemetry/exchange_online.py new file mode 100644 index 00000000..93013c9e --- /dev/null +++ b/telemetry/exchange_online.py @@ -0,0 +1,196 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Uber container section wrapping Mailbox, Calendar, Organization Apps, and Supported Email Clients/PSTs telemetry frames under a single Emails & Calendar card.""" + +import customtkinter as ctk +from telemetry.styles import * +from telemetry.mailbox_usage import MailboxUsageFrame +from telemetry.calendar_telemetry import CalendarTelemetryFrame +from telemetry.exchange_apps import ExchangeAppsFrame +from telemetry.email_client_support import EmailClientSupportFrame +from telemetry.exchange_connectors_ui import ExchangeConnectorsFrame +from telemetry.mail_security import MailSecurityFrame + +class ExchangeOnlineFrame(ctk.CTkFrame): + """Uber section container hosting Mailbox, Calendar, Apps, and Email Client telemetry frames vertically stacked.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None + + self.build_ui() + + def build_ui(self): + """Instantiates and stacks existing mailbox, calendar, apps, and email client support frames.""" + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + # Uber Title Heading + ctk.CTkLabel( + self.inner_pad, + text="Email, Calendar and Contacts", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN + ).pack(anchor="w", pady=(0, 5)) + + # Mailbox Usage Sub-frame + self.mailbox_view = MailboxUsageFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._check_overall_status, + concurrency_semaphore=self.semaphore + ) + self.mailbox_view.configure(fg_color="transparent", border_width=0) + self.mailbox_view.pack(fill="x", expand=True, pady=(0, 5)) + + # Calendar Telemetry Sub-frame + self.calendar_view = CalendarTelemetryFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._check_overall_status, + concurrency_semaphore=self.semaphore + ) + self.calendar_view.configure(fg_color="transparent", border_width=0) + self.calendar_view.pack(fill="x", expand=True, pady=(0, 5)) + + # Organization-wide Apps Sub-frame + self.apps_view = ExchangeAppsFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._check_overall_status, + concurrency_semaphore=self.semaphore + ) + self.apps_view.configure(fg_color="transparent", border_width=0) + self.apps_view.pack(fill="x", expand=True, pady=(0, 5)) + + # Mail Security Sub-frame + self.mail_security_view = MailSecurityFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._check_overall_status, + concurrency_semaphore=self.semaphore + ) + self.mail_security_view.configure(fg_color="transparent", border_width=0) + self.mail_security_view.pack(fill="x", expand=True, pady=(0, 5)) + + # Exchange Connectors Sub-frame + self.connectors_view = ExchangeConnectorsFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._check_overall_status, + concurrency_semaphore=self.semaphore + ) + self.connectors_view.configure(fg_color="transparent", border_width=0) + self.connectors_view.pack(fill="x", expand=True, pady=(0, 5)) + + # Email Client Support Sub-frame (combining supported email clients and PST files) + self.email_clients_view = EmailClientSupportFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._check_overall_status, + concurrency_semaphore=self.semaphore + ) + self.email_clients_view.configure(fg_color="transparent", border_width=0) + self.email_clients_view.pack(fill="x", expand=True, pady=(0, 5)) + + self.reset_view() + + def reset_view(self): + """Resets all sub-views and hides container.""" + self.pack_forget() + self.mailbox_view.reset_view() + self.calendar_view.reset_view() + self.apps_view.reset_view() + + if hasattr(self.mail_security_view, 'reset_view'): + self.mail_security_view.reset_view() + else: + self.mail_security_view.pack_forget() + + self.connectors_view.reset_view() + self.email_clients_view.reset_view() + + def trigger_fetch(self, tenant, client_id, client_secret): + """Displays container and delegates fetches to all sub-views.""" + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=10) + + self.mailbox_view.trigger_fetch(tenant, client_id, client_secret) + self.calendar_view.trigger_fetch(tenant, client_id, client_secret) + self.apps_view.trigger_fetch(tenant, client_id, client_secret) + + if hasattr(self.mail_security_view, 'trigger_fetch'): + self.mail_security_view.trigger_fetch(tenant, client_id, client_secret) + + self.connectors_view.trigger_fetch(tenant, client_id, client_secret) + self.email_clients_view.trigger_fetch(tenant, client_id, client_secret) + + def _check_overall_status(self): + """Updates main container status based on sub-frame statuses.""" + sub_statuses = [ + self.mailbox_view.status, + self.calendar_view.status, + self.apps_view.status, + getattr(self.mail_security_view, 'status', None), + self.connectors_view.status, + self.email_clients_view.status + ] + if "loading" in sub_statuses or getattr(self.mail_security_view, 'loading', False): + self.status = "loading" + elif "success" in sub_statuses: + self.status = "success" + else: + self.status = "error" + self.on_status_change() + + def cancel(self): + """Cancels all child views in this container.""" + self.mailbox_view.cancel() + self.calendar_view.cancel() + self.apps_view.cancel() + self.connectors_view.cancel() + self.email_clients_view.cancel() + + if hasattr(self.mail_security_view, 'cancel'): + self.mail_security_view.cancel() + + sub_statuses = [ + self.mailbox_view.status, + self.calendar_view.status, + self.apps_view.status, + self.connectors_view.status, + self.email_clients_view.status, + getattr(self.mail_security_view, 'status', None) + ] + if all(s is None for s in sub_statuses): + self.status = None + self.reset_view() + else: + self._check_overall_status() + diff --git a/telemetry/files_telemetry.py b/telemetry/files_telemetry.py new file mode 100644 index 00000000..6122f8cf --- /dev/null +++ b/telemetry/files_telemetry.py @@ -0,0 +1,118 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Uber container section wrapping SharePoint and OneDrive telemetry frames under a single Files card.""" + +import customtkinter as ctk +from telemetry.styles import * +from telemetry.sharepoint_onedrive_usage import SharePointUsageFrame, OneDriveUsageFrame + +class FilesTelemetryFrame(ctk.CTkFrame): + """Uber section container hosting SharePoint and OneDrive telemetry frames vertically stacked.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None + + self.build_ui() + + def build_ui(self): + """Instantiates and stacks existing SharePoint and OneDrive frames.""" + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + # Uber Title Heading + ctk.CTkLabel( + self.inner_pad, + text="Files", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN + ).pack(anchor="w", pady=(0, 5)) + + # SharePoint Usage Sub-frame + self.sharepoint_view = SharePointUsageFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._check_overall_status, + concurrency_semaphore=self.semaphore + ) + self.sharepoint_view.configure(fg_color="transparent", border_width=0) + self.sharepoint_view.pack(fill="x", expand=True, pady=(0, 5)) + + # Separator line + self.divider = ctk.CTkFrame(self.inner_pad, height=1, fg_color=COLOR_OUTLINE_LIGHT) + self.divider.pack(fill="x", pady=10) + + # OneDrive Usage Sub-frame + self.onedrive_view = OneDriveUsageFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._check_overall_status, + concurrency_semaphore=self.semaphore + ) + self.onedrive_view.configure(fg_color="transparent", border_width=0) + self.onedrive_view.pack(fill="x", expand=True, pady=(0, 5)) + + self.reset_view() + + def reset_view(self): + """Resets both sub-views and hides container.""" + self.pack_forget() + self.sharepoint_view.reset_view() + self.onedrive_view.reset_view() + self.divider.pack_forget() + + def trigger_fetch(self, tenant, client_id, client_secret): + """Displays container and delegates fetches to both sub-views.""" + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=10) + self.divider.pack(fill="x", pady=10) + + self.sharepoint_view.trigger_fetch(tenant, client_id, client_secret) + self.onedrive_view.trigger_fetch(tenant, client_id, client_secret) + + def _check_overall_status(self): + """Updates main container status based on sub-frame statuses.""" + if self.sharepoint_view.status == "loading" or self.onedrive_view.status == "loading": + self.status = "loading" + elif self.sharepoint_view.status == "success" or self.onedrive_view.status == "success": + self.status = "success" + else: + self.status = "error" + self.on_status_change() + + def cancel(self): + """Cancels all child views in this container.""" + self.sharepoint_view.cancel() + self.onedrive_view.cancel() + + sub_statuses = [ + self.sharepoint_view.status, + self.onedrive_view.status + ] + if all(s is None for s in sub_statuses): + self.status = None + self.reset_view() + else: + self._check_overall_status() diff --git a/telemetry/intune_policies.py b/telemetry/intune_policies.py new file mode 100644 index 00000000..d80a1c10 --- /dev/null +++ b/telemetry/intune_policies.py @@ -0,0 +1,266 @@ +import customtkinter as ctk +import threading +import logging +import traceback +import webbrowser +from telemetry.styles import * + +usage_logger = logging.getLogger("IntunePoliciesUI") + +def fetch_intune_policies_data(client_id, client_secret, tenant_id) -> dict: + """Fetch and parse Intune Configuration Policies and Device Configurations.""" + usage_logger.info("Starting Intune Policies fetch...") + try: + from core.graph.client import GraphClient + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=3, + backoff=2 + ) + client.authenticate() + session = client.token_manager.get_session() + + slot = client.token_manager.get_valid_token_slot() + access_token = slot["token"] + headers = {"Authorization": f"Bearer {access_token}", "Content-Type": "application/json"} + + device_configs = [] + try: + res1 = session.get("https://graph.microsoft.com/beta/deviceManagement/deviceConfigurations", headers=headers) + if res1.status_code == 200: + device_configs = res1.json().get("value", []) + except Exception as e: + usage_logger.warning(f"Failed to fetch deviceConfigurations: {e}") + + config_policies = [] + try: + res2 = session.get("https://graph.microsoft.com/beta/deviceManagement/configurationPolicies", headers=headers) + if res2.status_code == 200: + config_policies = res2.json().get("value", []) + except Exception as e: + usage_logger.warning(f"Failed to fetch configurationPolicies: {e}") + + client.token_manager.return_token_slot(slot) + client.close() + + import re + from collections import defaultdict + counts = defaultdict(int) + + def extract_platform_and_type(odata_type): + type_str = odata_type.replace("#microsoft.graph.", "") + platform = "Unknown" + + if type_str.startswith("windows10"): + platform = "Windows 10" + policy_type = type_str.replace("windows10", "") + elif type_str.startswith("windows"): + platform = "Windows" + policy_type = type_str.replace("windows", "") + elif type_str.startswith("ios"): + platform = "iOS" + policy_type = type_str.replace("ios", "") + elif type_str.startswith("android"): + platform = "Android" + policy_type = type_str.replace("android", "") + elif type_str.startswith("macOS"): + platform = "macOS" + policy_type = type_str.replace("macOS", "") + else: + policy_type = type_str + + if not policy_type: + policy_type = "Configuration" + + policy_type = re.sub(r"([A-Z])", r" \1", policy_type).strip() + return platform, policy_type + + for dc in device_configs: + platform, p_type = extract_platform_and_type(dc.get("@odata.type", "")) + counts[(platform, p_type)] += 1 + + for cp in config_policies: + platform = cp.get("platforms", "Unknown") + if platform == "windows10AndLater": + platform = "Windows 10" + elif platform == "windows81AndLater": + platform = "Windows 8.1" + elif platform == "macOS": + platform = "macOS" + else: + platform = platform.capitalize() + counts[(platform, "Settings Catalog")] += 1 + + rows = [] + for (platform, p_type), count in sorted(counts.items()): + rows.append((platform, p_type, str(count))) + + return { + "total_device_configs": len(device_configs), + "total_config_policies": len(config_policies), + "table_rows": rows, + "error": None + } + except Exception as e: + usage_logger.error("Failed to fetch Intune Policies", exc_info=True) + return {"error": str(e)} + +class IntunePoliciesFrame(ctk.CTkFrame): + """Component for rendering Intune Policies data.""" + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None + + self.build_ui() + + def build_ui(self): + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + self.header_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.header_frame.pack(fill="x", pady=(0, 10)) + + self.title_lbl = ctk.CTkLabel( + self.header_frame, + text="Intune Policies (Device Configurations)", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN + ) + self.title_lbl.pack(side="left", anchor="w") + + self.link_lbl = ctk.CTkLabel( + self.header_frame, + text="Open Intune Admin Center ↗", + font=FONT_BODY_BOLD, + text_color=COLOR_PRIMARY, + cursor="hand2" + ) + self.link_lbl.pack(side="left", anchor="w", padx=(15, 0)) + self.link_lbl.bind("", lambda e: webbrowser.open("https://intune.microsoft.com/#view/Microsoft_Intune_DeviceSettings/DevicesMenu/~/configuration")) + self.link_lbl.bind("", lambda e: self.link_lbl.configure(text_color=COLOR_PRIMARY_HOVER)) + self.link_lbl.bind("", lambda e: self.link_lbl.configure(text_color=COLOR_PRIMARY)) + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.reset_view() + + def reset_view(self): + self.pack_forget() + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=20) + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + ctk.CTkLabel(self.state_frame, text=f"✖ {error_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 10)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + usage_logger.info("Intune Policies trigger_fetch called.") + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=10) + self.grid_frame.pack_forget() + + self._set_state_loading("Scanning Microsoft Intune Device Configurations and Policies...") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str): + if self.semaphore: + self.semaphore.acquire() + try: + res = fetch_intune_policies_data(client_id, client_secret, tenant) + self.after(0, self._handle_result, res) + finally: + if self.semaphore: + self.semaphore.release() + + def _handle_result(self, result: dict): + self.state_frame.pack_forget() + for w in self.grid_frame.winfo_children(): + w.destroy() + + err = result.get("error") + if err: + self.status = "error" + if "Authorization_RequestDenied" in err or "403" in err: + self._set_state_error("Access Denied: Missing Intune or DeviceManagement permissions.") + else: + self._set_state_error(err) + else: + self.status = "success" + self.grid_frame.pack(fill="x", expand=True) + self._render_card(result) + + self.on_status_change() + + def _render_card(self, data: dict): + summary_frame = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + summary_frame.pack(fill="x", padx=10, pady=10) + + tot_dc = data.get("total_device_configs", 0) + tot_cp = data.get("total_config_policies", 0) + ctk.CTkLabel(summary_frame, text=f"Total Extracted: {tot_dc} Device Configurations | {tot_cp} Configuration Policies", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB).pack(anchor="w") + + metrics_grid = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_SURFACE, corner_radius=0) + metrics_grid.pack(fill="x", padx=10, pady=(5, 15)) + + headers = ["Platform", "Policy Type", "Number of Policies"] + for i in range(3): + metrics_grid.grid_columnconfigure(i, weight=1 if i == 2 else 2) + + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(metrics_grid, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + rows_data = data.get("table_rows", []) + + if not rows_data: + c = ctk.CTkFrame(metrics_grid, fg_color=COLOR_SURFACE, corner_radius=0) + c.grid(row=1, column=0, columnspan=3, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c, text="No policies detected.", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="center").pack(padx=10, pady=12) + return + + for r_idx, (platform, p_type, count) in enumerate(rows_data, start=1): + bg_style = COLOR_SURFACE if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + + vals = [platform, p_type, count] + + for c_idx, val in enumerate(vals): + c = ctk.CTkFrame(metrics_grid, fg_color=bg_style, corner_radius=0) + c.grid(row=r_idx, column=c_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c, text=val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left", wraplength=450).pack(padx=10, pady=12, anchor="nw") diff --git a/telemetry/m365_apps_telemetry.py b/telemetry/m365_apps_telemetry.py new file mode 100644 index 00000000..4f114ddb --- /dev/null +++ b/telemetry/m365_apps_telemetry.py @@ -0,0 +1,136 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Uber container section wrapping Active Users Usage, Active Users Trend, and M365 App Usage frames under a single M365 Apps card.""" + +import customtkinter as ctk +from telemetry.styles import * +from telemetry.active_users_usage import ActiveUsersUsageFrame, ActiveUsersTrendFrame, M365AppUsageFrame + +class M365AppsTelemetryFrame(ctk.CTkFrame): + """Uber section container hosting Active Users Usage, Trend, and M365 App Usage frames vertically stacked.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None + + self.build_ui() + + def build_ui(self): + """Instantiates and stacks existing usage sub-frames.""" + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + # Uber Title Heading + ctk.CTkLabel( + self.inner_pad, + text="M365 Apps", + font=FONT_HEADER_SMALL, + text_color=COLOR_TEXT_MAIN + ).pack(anchor="w", pady=(0, 5)) + + # Active Users Usage Sub-frame + self.active_users_view = ActiveUsersUsageFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._check_overall_status, + concurrency_semaphore=self.semaphore + ) + self.active_users_view.configure(fg_color="transparent", border_width=0) + self.active_users_view.pack(fill="x", expand=True, pady=(0, 5)) + + # Divider 1 + self.divider1 = ctk.CTkFrame(self.inner_pad, height=1, fg_color=COLOR_OUTLINE_LIGHT) + self.divider1.pack(fill="x", pady=10) + + # Active Users Trend Sub-frame + self.active_users_trend_view = ActiveUsersTrendFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._check_overall_status, + concurrency_semaphore=self.semaphore + ) + self.active_users_trend_view.configure(fg_color="transparent", border_width=0) + self.active_users_trend_view.pack(fill="x", expand=True, pady=(0, 5)) + + # Divider 2 + self.divider2 = ctk.CTkFrame(self.inner_pad, height=1, fg_color=COLOR_OUTLINE_LIGHT) + self.divider2.pack(fill="x", pady=10) + + # M365 App Usage Sub-frame + self.m365_apps_view = M365AppUsageFrame( + master=self.inner_pad, + log_callback=self.log_msg, + credentials_callback=self.get_credentials, + status_change_callback=self._check_overall_status, + concurrency_semaphore=self.semaphore + ) + self.m365_apps_view.configure(fg_color="transparent", border_width=0) + self.m365_apps_view.pack(fill="x", expand=True, pady=(0, 5)) + + self.reset_view() + + def reset_view(self): + """Resets all sub-views and hides container.""" + self.pack_forget() + self.active_users_view.reset_view() + self.active_users_trend_view.reset_view() + self.m365_apps_view.reset_view() + self.divider1.pack_forget() + self.divider2.pack_forget() + + def trigger_fetch(self, tenant, client_id, client_secret): + """Displays container and delegates fetches to all sub-views.""" + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=10) + self.divider1.pack(fill="x", pady=10) + self.divider2.pack(fill="x", pady=10) + + self.active_users_view.trigger_fetch(tenant, client_id, client_secret) + self.active_users_trend_view.trigger_fetch(tenant, client_id, client_secret) + self.m365_apps_view.trigger_fetch(tenant, client_id, client_secret) + + def _check_overall_status(self): + """Updates main container status based on sub-frame statuses.""" + sub_statuses = [self.active_users_view.status, self.active_users_trend_view.status, self.m365_apps_view.status] + if "loading" in sub_statuses: + self.status = "loading" + elif "success" in sub_statuses: + self.status = "success" + else: + self.status = "error" + self.on_status_change() + + def cancel(self): + """Cancels all child views in this container.""" + self.active_users_view.cancel() + self.active_users_trend_view.cancel() + self.m365_apps_view.cancel() + + sub_statuses = [self.active_users_view.status, self.active_users_trend_view.status, self.m365_apps_view.status] + if all(s is None for s in sub_statuses): + self.status = None + self.reset_view() + else: + self._check_overall_status() diff --git a/telemetry/m365_telemetry.py b/telemetry/m365_telemetry.py new file mode 100644 index 00000000..ae9eab9d --- /dev/null +++ b/telemetry/m365_telemetry.py @@ -0,0 +1,804 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Master coordinator tab and logging setup for Microsoft 365 tenant telemetry dashboard.""" + +import os +import sys +import time +import queue +import logging +import threading +from typing import Any, Dict, List, Optional +import customtkinter as ctk +from tkinter import messagebox +from logging.handlers import QueueHandler, QueueListener + +# Import modular view frames from their consolidated code/view modules +from telemetry.subscribed_skus import SubscribedSKUsFrame +from telemetry.directory import DirectoryFrame +from telemetry.m365_apps_telemetry import M365AppsTelemetryFrame +from telemetry.power_automate import PowerAutomateUsageFrame + + +# Import existing modular views +from telemetry.files_telemetry import FilesTelemetryFrame +from telemetry.exchange_online import ExchangeOnlineFrame +from telemetry.data_security_governance import DataSecurityGovernanceFrame +from telemetry.intune_policies import IntunePoliciesFrame + +from telemetry.styles import * + +# ================================================================================= +# ASYNC FILE LOGGING SETUP +# ================================================================================= + +_current_dir = os.path.dirname(os.path.abspath(__file__)) +_log_dir = os.path.join(_current_dir, 'logs') +os.makedirs(_log_dir, exist_ok=True) + +_log_queue = queue.Queue(-1) +_log_file_path = os.path.join(_log_dir, 'telemetry_log.txt') +_file_handler = logging.FileHandler(_log_file_path, mode='a', encoding='utf-8') +_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +_file_handler.setFormatter(_formatter) + +_queue_listener = QueueListener(_log_queue, _file_handler) +_queue_listener.start() + +# Configure the root logger to send all records to the log queue and clear other handlers (stdout/stderr) +root_logger = logging.getLogger() +root_logger.setLevel(logging.WARNING) +for h in list(root_logger.handlers): + root_logger.removeHandler(h) +root_logger.addHandler(QueueHandler(_log_queue)) + +# Set logger level to INFO for our own application packages/loggers +async_logger = logging.getLogger("M365TelemetryAsyncLogger") +async_logger.setLevel(logging.INFO) +async_logger.propagate = True + +logging.getLogger("core").setLevel(logging.INFO) +logging.getLogger("util").setLevel(logging.INFO) +logging.getLogger("chat").setLevel(logging.INFO) +logging.getLogger("PowerShellClient").setLevel(logging.INFO) + + +def update_log_directory(tenant_id: Optional[str] = None, client_id: Optional[str] = None) -> None: + """Updates the log directory dynamically once tenant and client ID are known, or reverts to default.""" + global _file_handler, _queue_listener + + _queue_listener.stop() + _file_handler.close() + + if tenant_id and client_id: + new_log_dir = os.path.join(_current_dir, 'logs', f"{tenant_id}_{client_id}") + else: + new_log_dir = os.path.join(_current_dir, 'logs') + + os.makedirs(new_log_dir, exist_ok=True) + new_log_file_path = os.path.join(new_log_dir, 'telemetry_log.txt') + + _file_handler = logging.FileHandler(new_log_file_path, mode='a', encoding='utf-8') + _file_handler.setFormatter(_formatter) + + _queue_listener = QueueListener(_log_queue, _file_handler) + _queue_listener.start() + + +# ================================================================================= +# MAIN TAB COORDINATOR +# ================================================================================= + +class M365TelemetryTab(ctk.CTkScrollableFrame): + """Encapsulates the UI coordinator for the Microsoft 365 Telemetry & Audit dashboard tab.""" + + def __init__(self, master, log_callback, retries_var, backoff_var, **kwargs): + super().__init__( + master, + fg_color="transparent", + scrollbar_button_color="white", + scrollbar_button_hover_color=COLOR_SECONDARY_HOVER, + **kwargs + ) + async_logger.info("Initializing M365TelemetryTab instance.") + + self.log_msg = log_callback + self.retries = retries_var + self.backoff = backoff_var + + self.lic_tenant_id = ctk.StringVar() + self.lic_client_ids = ctk.StringVar() + self.lic_client_secrets = ctk.StringVar() + + self.on_all_done_callback = None + self.telemetry_semaphore = threading.Semaphore(1) + self.is_fetching = False + + self.build_ui() + + # Batching orchestration for sequential loading of sections one by one + self.batches = [ + [self.subscribed_skus_view], + [self.directory_view], + [self.m365_apps_view], + [self.exchange_online_view], + [self.files_view], + [self.security_gov_view], + [self.intune_policies_view], + [self.power_automate_view] + ] + self.current_batch_index = 0 + + + # Bind mouse wheel globally to scroll this tab when hovered + self.bind_all("", self._handle_global_mousewheel, add="+") + self.bind_all("", self._handle_global_mousewheel, add="+") + self.bind_all("", self._handle_global_mousewheel, add="+") + + def _create_entry(self, parent, label, var, show=None): + f = ctk.CTkFrame(parent, fg_color="transparent") + f.pack(fill="x", pady=5) + ctk.CTkLabel(f, text=label, width=100, anchor="w", text_color=COLOR_TEXT_SUB).pack(side="left") + ctk.CTkEntry( + f, textvariable=var, show=show, height=40, corner_radius=4, + border_width=1, border_color=COLOR_OUTLINE, fg_color="transparent", + text_color=COLOR_TEXT_MAIN, + ).pack(side="left", fill="x", expand=True) + + def build_ui(self): + async_logger.info("Building graphical UI elements for M365 Telemetry Tab.") + + ctk.CTkLabel(self, text="Connect your Microsoft Azure account to authenticate and audit tenant licensing bundle inventories and usage.", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB).pack(anchor="w", pady=(0, 15)) + + self.inputs_frame = ctk.CTkFrame(self, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + self.inputs_frame.pack(fill="x", pady=5) + + inner_pad = ctk.CTkFrame(self.inputs_frame, fg_color="transparent") + inner_pad.pack(fill="x", padx=15, pady=15) + + self._create_entry(inner_pad, "Tenant ID", self.lic_tenant_id) + self._create_entry(inner_pad, "Client ID", self.lic_client_ids) + self._create_entry(inner_pad, "Client Secret", self.lic_client_secrets, show="*") + + actions_frame = ctk.CTkFrame(self, fg_color="transparent") + actions_frame.pack(fill="x", pady=(20, 10)) + + self.btn_lic_submit = ctk.CTkButton( + actions_frame, text="Submit", width=160, height=40, corner_radius=20, + font=FONT_BODY_BOLD, fg_color=COLOR_PRIMARY, hover_color=COLOR_PRIMARY_HOVER, + command=self.authenticate_licenses_tab, + ) + self.btn_lic_submit.pack(side="left") + + self.lbl_lic_status = ctk.CTkLabel(actions_frame, text="", font=FONT_BODY_MEDIUM) + self.lbl_lic_status.pack(side="left", padx=20) + + # ---------------------------------------------------- + # MODULAR UI SECTIONS + # ---------------------------------------------------- + + # 1. Subscribed SKUs Section + self.subscribed_skus_view = SubscribedSKUsFrame( + master=self, + log_callback=self.log_msg, + credentials_callback=self._get_credentials, + status_change_callback=self._check_all_done, + retries_var=self.retries, + backoff_var=self.backoff, + concurrency_semaphore=self.telemetry_semaphore + ) + + # 1b. Directory Groups Section + self.directory_view = DirectoryFrame( + master=self, + log_callback=self.log_msg, + credentials_callback=self._get_credentials, + status_change_callback=self._check_all_done, + retries_var=self.retries, + backoff_var=self.backoff, + concurrency_semaphore=self.telemetry_semaphore + ) + + # 2. M365 Apps Section (Uber Container) + self.m365_apps_view = M365AppsTelemetryFrame( + master=self, + log_callback=self.log_msg, + credentials_callback=self._get_credentials, + status_change_callback=self._check_all_done, + concurrency_semaphore=self.telemetry_semaphore + ) + + # 5. Exchange Online Usage Section + self.exchange_online_view = ExchangeOnlineFrame( + master=self, + log_callback=self.log_msg, + credentials_callback=self._get_credentials, + status_change_callback=self._check_all_done, + concurrency_semaphore=self.telemetry_semaphore + ) + + + + # 5c. Files (SharePoint & OneDrive) Section + self.files_view = FilesTelemetryFrame( + master=self, + log_callback=self.log_msg, + credentials_callback=self._get_credentials, + status_change_callback=self._check_all_done, + concurrency_semaphore=self.telemetry_semaphore + ) + + # 6. Data Security & Governance Section + self.security_gov_view = DataSecurityGovernanceFrame( + master=self, + log_callback=self.log_msg, + credentials_callback=self._get_credentials, + status_change_callback=self._check_all_done, + concurrency_semaphore=self.telemetry_semaphore + ) + + # 6.5. Intune Policies Section + self.intune_policies_view = IntunePoliciesFrame( + master=self, + log_callback=self.log_msg, + credentials_callback=self._get_credentials, + status_change_callback=self._check_all_done, + concurrency_semaphore=self.telemetry_semaphore + ) + + # 7. Power Automate Section + self.power_automate_view = PowerAutomateUsageFrame( + master=self, + log_callback=self.log_msg, + credentials_callback=self._get_credentials, + status_change_callback=self._check_all_done, + concurrency_semaphore=self.telemetry_semaphore + ) + + self._hide_all_grids() + + # Wrap all leaf views to support cancellation and prevent race conditions + for leaf in self._get_all_leaf_views(): + self._wrap_view_for_cancellation(leaf) + + def _hide_all_grids(self): + views = [ + self.subscribed_skus_view, + self.directory_view, + self.m365_apps_view, + self.exchange_online_view, + self.files_view, + self.security_gov_view, + self.intune_policies_view, + self.power_automate_view + ] + for view in views: + view.reset_view() + view.status = None + + def reset_tab(self): + """Resets the coordinator status, credentials variables, submission button, and hides all grids.""" + async_logger.info("Resetting M365TelemetryTab coordinator and hiding all sub-grids.") + self.lic_tenant_id.set("") + self.lic_client_ids.set("") + self.lic_client_secrets.set("") + self.btn_lic_submit.configure(state="normal", text="Submit") + self.lbl_lic_status.configure(text="") + self.current_batch_index = 0 + self._hide_all_grids() + + def _get_credentials(self): + tenant = self.lic_tenant_id.get().strip() + client_str = self.lic_client_ids.get().strip() + secret_str = self.lic_client_secrets.get().strip() + + if not tenant or not client_str or not secret_str: + return None, None, None + + clients = [x.strip() for x in client_str.split(",") if x.strip()] + secrets = [x.strip() for x in secret_str.split(",") if x.strip()] + return tenant, clients, secrets + + def _check_all_done(self): + """Checks if all sections of the current batch have resolved, then triggers next batch or finishes.""" + if not getattr(self, "is_fetching", False): + # If fetching was cancelled, ignore any further background thread completions + return + + if not hasattr(self, "batches"): + return + + current_views = self.batches[self.current_batch_index] + batch_states = [view.status for view in current_views] + + if "loading" in batch_states: + return + + # Current batch completed. Check if there are more batches to run + if self.current_batch_index < len(self.batches) - 1: + self.current_batch_index += 1 + self.trigger_current_batch() + return + + # All batches have finished. Make sure no individual retries are still loading + all_views = [view for batch in self.batches for view in batch] + global_states = [v.status for v in all_views] + if "loading" in global_states: + return + + # Re-enable the submit button + self.is_fetching = False + self.btn_lic_submit.configure(state="normal", text="Submit") + + success = all(s == "success" for s in global_states) + if success: + self.lbl_lic_status.configure(text="✔ All Inventory and Usage Reports Pulled Successfully!", text_color=COLOR_SUCCESS) + else: + self.lbl_lic_status.configure(text="⚠ Some reports failed. Please retry individually.", text_color=COLOR_ERROR) + + if hasattr(self, "on_all_done_callback") and self.on_all_done_callback: + self.on_all_done_callback(success) + + def trigger_current_batch(self): + """Triggers the fetches for the current batch of sections.""" + tenant, clients, secrets = self._get_credentials() + if not tenant: + return + + current_views = self.batches[self.current_batch_index] + async_logger.info(f"Triggering batch {self.current_batch_index + 1} with {len(current_views)} views.") + + for view in current_views: + if isinstance(view, SubscribedSKUsFrame): + view.trigger_fetch(tenant, clients, secrets) + else: + view.trigger_fetch(tenant, clients[0], secrets[0]) + + def authenticate_licenses_tab(self): + """Master full sequential fetch of sections, or cancel if already fetching.""" + if getattr(self, "is_fetching", False): + self.cancel_fetching() + return + + async_logger.info("Master Submit triggered. Restarting all fetches sequentially.") + + tenant, clients, secrets = self._get_credentials() + if not tenant: + async_logger.warning("Authentication aborted: Missing credential parameters.") + messagebox.showerror("Credential Error", "Please provide complete Tenant ID, Client ID, and Client Secret strings.", parent=self) + return + + self.is_fetching = True + self.btn_lic_submit.configure(state="normal", text="Cancel") + self.lbl_lic_status.configure(text="Querying Microsoft Graph APIs and Reports sequentially...", text_color=COLOR_TEXT_SUB) + + # Reset all views first + self._hide_all_grids() + + self.current_batch_index = 0 + self.trigger_current_batch() + + def cancel_fetching(self): + """Cancels the current fetching process and stops all subsequent batches.""" + async_logger.info("Cancellation triggered by user.") + self.is_fetching = False + + self.btn_lic_submit.configure(state="normal", text="Submit") + self.lbl_lic_status.configure(text="✖ Query cancelled by user.", text_color=COLOR_ERROR) + + # Propagate cancel to all top-level views + for batch in self.batches: + for view in batch: + if hasattr(view, "cancel"): + view.cancel() + + if hasattr(self, "on_all_done_callback") and self.on_all_done_callback: + self.on_all_done_callback(False) + + def _get_all_leaf_views(self): + """Returns a list of all active leaf/base telemetry views across all cards.""" + return [ + self.subscribed_skus_view, + self.directory_view, + self.m365_apps_view.active_users_view, + self.m365_apps_view.active_users_trend_view, + self.m365_apps_view.m365_apps_view, + self.exchange_online_view.mailbox_view, + self.exchange_online_view.calendar_view, + self.exchange_online_view.apps_view, + self.exchange_online_view.mail_security_view, + self.exchange_online_view.connectors_view, + self.exchange_online_view.email_clients_view, + self.files_view.sharepoint_view, + self.files_view.onedrive_view, + self.security_gov_view, + self.intune_policies_view, + self.power_automate_view + ] + + def _find_main_title_label(self, view): + """Recursively searches the widget tree to identify the exact header/title label of a leaf view.""" + known_titles = { + "SubscribedSKUsFrame": "Subscribed SKUs", + "DirectoryFrame": "Directory Summary", + "ActiveUsersUsageFrame": "Active Users Usage", + "ActiveUsersTrendFrame": "Active Users Trend", + "M365AppUsageFrame": "M365 App Usage", + "MailboxUsageFrame": "Mailbox Usage Summary", + "CalendarTelemetryFrame": "Calendar & Room Resource Telemetry", + "ExchangeAppsFrame": "Exchange Integrated Apps", + "ExchangeConnectorsFrame": "Exchange Connectors (Inbound & Outbound Routing)", + "MailSecurityFrame": "Mail Security", + "EmailClientSupportFrame": "Email Client Classification", + "SharePointUsageFrame": "SharePoint Online Sites & Files Summary", + "OneDriveUsageFrame": "OneDrive for Business Personal Accounts Summary", + "DataSecurityGovernanceFrame": "Data Security & Governance", + "IntunePoliciesFrame": "Intune Policies (Device Configurations)", + "PowerAutomateUsageFrame": "Power Automate (Workflows & Flows)" + } + + target_text = known_titles.get(view.__class__.__name__, "") + + def search(widget): + if isinstance(widget, ctk.CTkLabel): + try: + text = widget.cget("text") + if text == target_text or (target_text and target_text in text): + return widget + except Exception: + pass + if hasattr(widget, "winfo_children"): + for child in widget.winfo_children(): + res = search(child) + if res: + return res + return None + + lbl = search(view) + if lbl: + return lbl + + # Fallback: search for the first CTkLabel + def first_label(widget): + if isinstance(widget, ctk.CTkLabel): + return widget + if hasattr(widget, "winfo_children"): + for child in widget.winfo_children(): + res = first_label(child) + if res: + return res + return None + return first_label(view) + + def _wrap_view_for_cancellation(self, view): + """Wraps a leaf view's trigger, render/handle, reset and after methods dynamically to enforce thread safety, cancellation, stale thread filtering, and precise execution time tracking.""" + view.current_request_id = 0 + view.is_cancelled = False + view.fetch_time_lbl = None + view.fetch_start_time = 0.0 + + orig_trigger = view.trigger_fetch + orig_reset = view.reset_view + orig_after = view.after + + # Wrap the semaphore if present to capture the exact start time after acquisition + if getattr(view, "semaphore", None): + orig_sem = view.semaphore + + class WrappedSemaphore: + def __init__(self, sem): + self._sem = sem + def acquire(self, *args, **kwargs): + res = self._sem.acquire(*args, **kwargs) + cur_thread = threading.current_thread() + sub_sec = getattr(cur_thread, "sub_section", None) + if sub_sec: + view.sub_section_start_times[sub_sec] = time.time() + else: + view.fetch_start_time = time.time() + return res + def release(self, *args, **kwargs): + return self._sem.release(*args, **kwargs) + def __getattr__(self, name): + return getattr(self._sem, name) + + view.semaphore = WrappedSemaphore(orig_sem) + + def display_fetch_time(elapsed): + # Destroy old label if exists + if hasattr(view, "fetch_time_lbl") and view.fetch_time_lbl: + try: + view.fetch_time_lbl.destroy() + except Exception: + pass + view.fetch_time_lbl = None + + # Create a new floating label next to the title + view.fetch_time_lbl = ctk.CTkLabel( + view, + text=f"⏱ {elapsed:.2f}s", + font=ctk.CTkFont(family="Segoe UI", size=11, weight="bold"), + text_color=COLOR_PRIMARY + ) + + # Place in the center of the header frame if available (for Subscribed SKUs Frame) + if hasattr(view, "lic_header") and view.lic_header: + view.fetch_time_lbl.place( + in_=view.lic_header, + relx=0.5, + rely=0.5, + anchor="center" + ) + else: + # Otherwise, place at the exact horizontal center of the card container, aligned vertically with the title + view.fetch_time_lbl.place(relx=0.5, rely=0.0, anchor="center", y=32) + + def display_sub_section_time(sub_sec, header_frame, elapsed): + # Destroy old label if exists + if hasattr(view, "sub_section_timer_labels") and sub_sec in view.sub_section_timer_labels: + lbl = view.sub_section_timer_labels[sub_sec] + if lbl: + try: + lbl.destroy() + except Exception: + pass + view.sub_section_timer_labels[sub_sec] = None + + # Create a new floating label + lbl = ctk.CTkLabel( + header_frame, + text=f"⏱ {elapsed:.2f}s", + font=ctk.CTkFont(family="Segoe UI", size=11, weight="bold"), + text_color=COLOR_PRIMARY + ) + + # Place in the exact center of the sub-section's header frame + # (completely safe: title/links on the left, export button on the right) + lbl.place( + in_=header_frame, + relx=0.5, + rely=0.5, + anchor="center" + ) + + view.sub_section_timer_labels[sub_sec] = lbl + + # Determine which rendering method is present + has_render = hasattr(view, "_render_success") and hasattr(view, "_render_error") + has_handle = hasattr(view, "_handle_result") + is_security_gov = hasattr(view, "_handle_labels_result") + + if is_security_gov: + view.sub_section_start_times = {} + view.sub_section_timer_labels = {} + + orig_labels_handle = view._handle_labels_result + orig_retention_handle = view._handle_retention_result + orig_auth_handle = view._handle_auth_result + + def new_labels_handle(*args, **kwargs): + if view.is_cancelled: + return + elapsed = time.time() - view.sub_section_start_times.get("labels", time.time()) + display_sub_section_time("labels", view.labels_header_frame, elapsed) + orig_labels_handle(*args, **kwargs) + + def new_retention_handle(*args, **kwargs): + if view.is_cancelled: + return + elapsed = time.time() - view.sub_section_start_times.get("retention", time.time()) + display_sub_section_time("retention", view.retention_header_frame, elapsed) + orig_retention_handle(*args, **kwargs) + + def new_auth_handle(*args, **kwargs): + if view.is_cancelled: + return + elapsed = time.time() - view.sub_section_start_times.get("auth", time.time()) + display_sub_section_time("auth", view.auth_header_frame, elapsed) + orig_auth_handle(*args, **kwargs) + + view._handle_labels_result = new_labels_handle + view._handle_retention_result = new_retention_handle + view._handle_auth_result = new_auth_handle + + if has_render: + orig_success = view._render_success + orig_error = view._render_error + + def new_success(*args, **kwargs): + if view.is_cancelled: + async_logger.info(f"Ignored _render_success for {view.__class__.__name__} because it was cancelled.") + return + elapsed = time.time() - getattr(view, "fetch_start_time", time.time()) + display_fetch_time(elapsed) + orig_success(*args, **kwargs) + + def new_error(*args, **kwargs): + if view.is_cancelled: + async_logger.info(f"Ignored _render_error for {view.__class__.__name__} because it was cancelled.") + return + elapsed = time.time() - getattr(view, "fetch_start_time", time.time()) + display_fetch_time(elapsed) + orig_error(*args, **kwargs) + + view._render_success = new_success + view._render_error = new_error + + elif has_handle: + orig_handle = view._handle_result + + def new_handle(*args, **kwargs): + if view.is_cancelled: + async_logger.info(f"Ignored _handle_result for {view.__class__.__name__} because it was cancelled.") + return + elapsed = time.time() - getattr(view, "fetch_start_time", time.time()) + display_fetch_time(elapsed) + orig_handle(*args, **kwargs) + + view._handle_result = new_handle + + def new_trigger(*args, **kwargs): + if hasattr(view, "fetch_time_lbl") and view.fetch_time_lbl: + try: + view.fetch_time_lbl.destroy() + except Exception: + pass + view.fetch_time_lbl = None + + if hasattr(view, "sub_section_timer_labels"): + for lbl in view.sub_section_timer_labels.values(): + if lbl: + try: + lbl.destroy() + except Exception: + pass + view.sub_section_timer_labels.clear() + + if is_security_gov: + view.sub_section_start_times.clear() + + # Only set fetch_start_time here if there is no semaphore (otherwise WrappedSemaphore handles it) + if not getattr(view, "semaphore", None): + view.fetch_start_time = time.time() + + view.current_request_id += 1 + view.is_cancelled = False + + # Temporarily tag spawned threads with the current request ID and sub-section + orig_thread_init = threading.Thread.__init__ + req_id = view.current_request_id + + def new_thread_init(thread_self, *t_args, **t_kwargs): + orig_thread_init(thread_self, *t_args, **t_kwargs) + thread_self.request_id = req_id + + # Tag with sub-section name based on target method name + target = t_kwargs.get("target") or (t_args[0] if t_args else None) + if target: + target_name = getattr(target, "__name__", "") + if "labels" in target_name: + thread_self.sub_section = "labels" + elif "retention" in target_name: + thread_self.sub_section = "retention" + elif "auth" in target_name: + thread_self.sub_section = "auth" + + threading.Thread.__init__ = new_thread_init + try: + orig_trigger(*args, **kwargs) + finally: + threading.Thread.__init__ = orig_thread_init + + def new_reset(*args, **kwargs): + view.is_cancelled = True + if hasattr(view, "fetch_time_lbl") and view.fetch_time_lbl: + try: + view.fetch_time_lbl.destroy() + except Exception: + pass + view.fetch_time_lbl = None + + if hasattr(view, "sub_section_timer_labels"): + for lbl in view.sub_section_timer_labels.values(): + if lbl: + try: + lbl.destroy() + except Exception: + pass + view.sub_section_timer_labels.clear() + orig_reset(*args, **kwargs) + + def new_after(ms, callback, *args, **kwargs): + # Identify the calling thread + cur_thread = threading.current_thread() + thread_req_id = getattr(cur_thread, "request_id", None) + + # If the calling thread has a request_id and it doesn't match the current one, discard it + if thread_req_id is not None and thread_req_id != view.current_request_id: + async_logger.warning( + f"Discarded after() callback for {view.__class__.__name__} from stale thread " + f"(thread req_id: {thread_req_id}, current req_id: {view.current_request_id})" + ) + return None + return orig_after(ms, callback, *args, **kwargs) + + def cancel_method(): + view.is_cancelled = True + view.current_request_id += 1 + if view.status == "loading": + view.status = None + view.reset_view() + + view.trigger_fetch = new_trigger + view.reset_view = new_reset + view.after = new_after + view.cancel = cancel_method + + def get_all_telemetry_data(self) -> dict: + """Retrieves cached telemetry data and charts from all sub-views.""" + return { + "tenant_id": self.lic_tenant_id.get().strip(), + "skus": getattr(self.subscribed_skus_view, "last_licenses_items", []), + "directory": { + "domains": getattr(self.directory_view, "last_domains", []), + "group_counts": getattr(self.directory_view, "last_group_counts", {}), + "user_counts": getattr(self.directory_view, "last_user_counts", {}) + }, + "o365_usage": getattr(self.m365_apps_view.active_users_view, "o365_data", []), + "o365_trend": getattr(self.m365_apps_view.active_users_trend_view, "trend_data", {}), + "m365_apps": getattr(self.m365_apps_view.m365_apps_view, "m365_data", []), + "mailbox": getattr(self.exchange_online_view.mailbox_view, "last_data", {}), + "calendar": getattr(self.exchange_online_view.calendar_view, "last_data", {}), + "sharepoint": getattr(self.files_view.sharepoint_view, "last_data", {}), + "onedrive": getattr(self.files_view.onedrive_view, "last_data", {}), + "security_labels": getattr(self.security_gov_view, "last_labels_data", []), + "retention_policies": getattr(self.security_gov_view, "last_policies_data", []), + "power_automate": getattr(self.power_automate_view, "last_results", {}) + } + + def is_descendant(self, parent, widget) -> bool: + """Recursively checks if a widget (or its Tkinter path name) is a descendant of parent.""" + if not widget: + return False + if isinstance(widget, str): + try: + widget = self.nametowidget(widget) + except Exception: + return False + if widget == parent: + return True + if hasattr(widget, "master") and widget.master is not None: + return self.is_descendant(parent, widget.master) + return False + + def _handle_global_mousewheel(self, event): + """Redirects mousewheel scrolling to the tab's parent canvas if hovered.""" + try: + widget = self.winfo_containing(event.x_root, event.y_root) + except Exception: + return + + if self.is_descendant(self, widget): + if event.num == 4: # Linux scroll up + self._parent_canvas.yview("scroll", -1, "units") + elif event.num == 5: # Linux scroll down + self._parent_canvas.yview("scroll", 1, "units") + else: # Windows / macOS + if sys.platform == "darwin": + # macOS trackpad/mouse delta + self._parent_canvas.yview("scroll", -event.delta, "units") + else: + # Windows delta (usually multiple of 120) + self._parent_canvas.yview("scroll", -int(event.delta / 120), "units") diff --git a/telemetry/mail_security.py b/telemetry/mail_security.py new file mode 100644 index 00000000..5ecfc178 --- /dev/null +++ b/telemetry/mail_security.py @@ -0,0 +1,198 @@ +import customtkinter as ctk +import time +import requests +from telemetry.styles import * + +class MailSecurityFrame(ctk.CTkFrame): + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color="transparent", **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change_cb = status_change_callback + + self.status = None + self.loading = True + self.error_msg = None + + self.total_eop_users = 0 + self.eop_skus = [] + + self.total_defender_users = 0 + self.defender_skus = [] + + self.last_data = {} + + self.grid_frame = ctk.CTkFrame(self, fg_color=COLOR_SURFACE, corner_radius=12, border_width=1, border_color=COLOR_OUTLINE_LIGHT) + + header = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + header.pack(fill="x", padx=15, pady=(15, 5)) + ctk.CTkLabel(header, text="Mail Security", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(side="left") + + self.loading_label = ctk.CTkLabel(self.grid_frame, text="Loading Mail Security Data...", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB) + self.loading_label.pack(pady=20) + + def trigger_fetch(self, tenant, client_id, client_secret): + self.pack(fill="x", expand=True, pady=(0, 5)) + self.status = "loading" + self.loading = True + self.on_status_change() + import threading + threading.Thread(target=self._fetch_data, args=(tenant, client_id, client_secret), daemon=True).start() + + def _fetch_data(self, tenant, c_id, c_secret): + try: + if not tenant or not c_id or not c_secret: + self.error_msg = "Missing credentials." + self.loading = False + self.on_status_change() + return + + from util.auth_manager import TokenManager + tm = TokenManager(tenant_id=tenant, client_ids=[c_id], client_secrets=[c_secret], concurrency=1, retries=1, backoff=0) + tm.authenticate_all() + + slot = tm.get_valid_token_slot() + if not slot: + self.error_msg = "Authentication failed: No valid token." + self.loading = False + self.on_status_change() + return + + token = slot["token"] + headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} + + url = "https://graph.microsoft.com/v1.0/subscribedSkus" + res = requests.get(url, headers=headers, timeout=15) + + tm.return_token_slot(slot) + + if res.status_code != 200: + self.error_msg = f"Graph API Error {res.status_code}: {res.text}" + self.loading = False + self.on_status_change() + return + + data = res.json().get("value", []) + + defender_skus_set = set() + eop_skus_set = set() + + defender_users = 0 + eop_users = 0 + + for sku in data: + raw_part_num = sku.get("skuPartNumber", "Unknown") + if isinstance(raw_part_num, list): + part_num = ", ".join([str(x) for x in raw_part_num]) + else: + part_num = str(raw_part_num) + + consumed = int(sku.get("consumedUnits", 0)) + plans = sku.get("servicePlans", []) + + has_defender = False + has_eop = False + + for p in plans: + if p.get("provisioningStatus") == "Success": + name = p.get("servicePlanName", "").upper() + if "DEFENDER_PLATFORM_FOR_OFFICE" in name or "ATP_ENTERPRISE" in name: + has_defender = True + elif "EXCHANGE_S_ENTERPRISE" in name or "EXCHANGE_S_STANDARD" in name or "EXCHANGE_S_FOUNDATION" in name: + has_eop = True + + if has_defender: + defender_skus_set.add(part_num) + defender_users += consumed + elif has_eop: + eop_skus_set.add(part_num) + eop_users += consumed + + self.defender_skus = list(defender_skus_set) + self.total_defender_users = defender_users + + self.eop_skus = list(eop_skus_set) + self.total_eop_users = eop_users + + self.last_data = { + "defender": {"skus": self.defender_skus, "users": self.total_defender_users}, + "eop": {"skus": self.eop_skus, "users": self.total_eop_users} + } + + self.status = "success" + self.loading = False + self.on_status_change() + self.on_status_change_cb() + + except Exception as e: + self.error_msg = f"Exception: {str(e)}" + self.status = "error" + self.loading = False + self.on_status_change() + self.on_status_change_cb() + + def reset_view(self): + self.pack_forget() + self.status = None + self.error_msg = None + self.loading = True + self.grid_frame.pack_forget() + for widget in self.grid_frame.winfo_children(): + if widget != self.grid_frame.winfo_children()[0]: # Keep header + widget.destroy() + self.loading_label = ctk.CTkLabel(self.grid_frame, text="Loading Mail Security Data...", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB) + self.loading_label.pack(pady=20) + + def cancel(self): + self.status = None + self.loading = False + self.reset_view() + + def on_status_change(self): + try: + self.loading_label.destroy() + except: + pass + + if self.loading: + return + + if self.error_msg: + ctk.CTkLabel(self.grid_frame, text=self.error_msg, text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM).pack(pady=10) + self.grid_frame.pack(fill="x", expand=True, pady=10) + return + + metrics_grid = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_SURFACE, corner_radius=0) + metrics_grid.pack(fill="x", padx=10, pady=(5, 5)) + + headers = ["Mail Security Configuration", "Detected SKUs", "Affected Users"] + for i in range(3): + metrics_grid.grid_columnconfigure(i, weight=1 if i == 2 else 2) + + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(metrics_grid, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + rows_data = [] + if self.defender_skus: + rows_data.append(("Microsoft Defender for Office 365", ", ".join(self.defender_skus), str(self.total_defender_users))) + if self.eop_skus: + rows_data.append(("Exchange Online Protection (Baseline)", ", ".join(self.eop_skus), str(self.total_eop_users))) + + if not rows_data: + c = ctk.CTkFrame(metrics_grid, fg_color=COLOR_SURFACE, corner_radius=0) + c.grid(row=1, column=0, columnspan=3, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c, text="No Mail Security SKUs detected.", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="center").pack(padx=10, pady=12) + else: + for r_idx, vals in enumerate(rows_data, start=1): + bg_style = COLOR_SURFACE if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + for c_idx, val in enumerate(vals): + c = ctk.CTkFrame(metrics_grid, fg_color=bg_style, corner_radius=0) + c.grid(row=r_idx, column=c_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c, text=val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, justify="left", wraplength=450).pack(padx=10, pady=12, anchor="nw") + + disclaimer = ctk.CTkLabel(self.grid_frame, text="Note: Users can track outbound connectors displayed below to identify 3rd-party security apps.", font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_SUB, anchor="w") + disclaimer.pack(fill="x", padx=15, pady=(5, 15)) + self.grid_frame.pack(fill="x", expand=True, pady=10) diff --git a/telemetry/mailbox_usage.py b/telemetry/mailbox_usage.py new file mode 100644 index 00000000..7dd1566b --- /dev/null +++ b/telemetry/mailbox_usage.py @@ -0,0 +1,358 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Modular Exchange Online Mailbox usage telemetry scanners and visual interfaces.""" + +import os +import logging +import threading +import pandas as pd +import customtkinter as ctk + +# Import unified core service layer +from core.graph.client import GraphClient +from core.graph.reports import ReportsService +from core.graph.directory import DirectoryService +from core.powershell.client import PowerShellClient +from core.powershell.mailbox import MailboxStatsService + +# Bind to the async logger initialized in m365_telemetry.py +usage_logger = logging.getLogger("M365TelemetryAsyncLogger") + +# ================================================================================= +# CONSTANTS & STYLES (Imported from shared styles) +# ================================================================================= +from telemetry.styles import * + +# ================================================================================= +# PIPELINE UTILITIES +# ================================================================================= + +def format_bytes(num_bytes: float) -> str: + """Formats raw byte values into highly readable string equivalents (e.g., GB, TB).""" + if num_bytes is None: + return "0.00 Bytes" + + for unit in ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB']: + if num_bytes < 1024.0: + return f"{num_bytes:,.2f} {unit}" + num_bytes /= 1024.0 + return f"{num_bytes:,.2f} EB" + +def parse_mailbox_usage_csv(filepath: str) -> dict: + """Streams the Mailbox Usage Detail CSV and aggregates metrics in chunks using pandas.""" + usage_logger.info(f"Processing Mailbox Usage file: {os.path.basename(filepath)}") + if not os.path.exists(filepath): + usage_logger.error(f"Error: Could not find Mailbox report {filepath}") + raise FileNotFoundError("Mailbox Usage report not found.") + + cols = ["Storage Used (Byte)", "Item Count"] + headers = pd.read_csv(filepath, nrows=0).columns.tolist() + if "Is Deleted" in headers: + cols.append("Is Deleted") + + total_mailboxes = 0 + total_bytes = 0 + total_emails = 0 + + for chunk in pd.read_csv(filepath, usecols=cols, chunksize=10000): + if "Is Deleted" in chunk.columns: + active_chunk = chunk[~chunk["Is Deleted"].astype(str).str.strip().str.upper().isin(["TRUE", "1"])] + else: + active_chunk = chunk + + active_chunk = active_chunk.dropna(subset=['Storage Used (Byte)', 'Item Count']) + + total_mailboxes += len(active_chunk) + total_bytes += int(active_chunk['Storage Used (Byte)'].sum()) + total_emails += int(active_chunk['Item Count'].sum()) + + avg_bytes = (total_bytes / total_mailboxes) if total_mailboxes > 0 else 0.0 + avg_emails = (total_emails / total_mailboxes) if total_mailboxes > 0 else 0.0 + + usage_logger.info( + f"Mailbox parsing complete: mailboxes={total_mailboxes}, " + f"storage={format_bytes(total_bytes)}, items={total_emails}" + ) + + return { + "total_mailboxes": total_mailboxes, + "total_storage_bytes": total_bytes, + "total_storage_formatted": format_bytes(total_bytes), + "average_mailbox_size_bytes": avg_bytes, + "average_mailbox_size_formatted": format_bytes(avg_bytes), + "total_emails": total_emails, + "average_emails": avg_emails + } + +def run_mailbox_usage_pipeline(client_id: str, client_secret: str, tenant_id: str) -> dict: + """Pipeline specifically for Mailbox Usage telemetry data collection.""" + usage_logger.info("Starting Mailbox Usage Telemetry Pipeline...") + + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate() + service = ReportsService(client) + + tenant_domain = tenant_id + try: + dir_svc = DirectoryService(client) + tenant_domain = dir_svc.get_tenant_primary_domain() + usage_logger.info(f"Retrieved primary tenant domain for Connect-ExchangeOnline: {tenant_domain}") + except Exception as e: + usage_logger.warning(f"Could not retrieve tenant domain via Graph. Falling back to Tenant ID Guid: {e}") + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + reports_dir = os.path.join(script_dir, "reports", f"{tenant_id}_{client_id}") + + service.download_mailbox_usage_detail(reports_dir) + usage_logger.info("Mailbox Usage CSV download completed. Initiating parser...") + client.close() + + data = parse_mailbox_usage_csv(os.path.join(reports_dir, "MailboxUsageDetail(180d).csv")) + + shared_count = None + shared_bytes = None + pf_count = None + pf_bytes = None + mail_pf_count = None + powershell_error = None + + try: + usage_logger.info("Running PowerShell script for Shared Mailboxes and Public Folders stats...") + ps_client = PowerShellClient( + tenant_id=tenant_domain, + client_id=client_id, + client_secret=client_secret, + cert_tenant_id=tenant_id + ) + pb_service = MailboxStatsService(ps_client) + stats = pb_service.fetch_mailbox_and_folder_stats() + + shared_count = stats.get("SharedMailboxesCount") + shared_bytes = stats.get("SharedMailboxesTotalBytes") + pf_count = stats.get("PublicFoldersCount") + pf_bytes = stats.get("PublicFoldersTotalBytes") + mail_pf_count = stats.get("MailPublicFoldersCount") + + # Parse and log individual command errors + errors = stats.get("Errors", {}) + if errors: + for component, err_msg in errors.items(): + usage_logger.error(f"PowerShell error querying {component}: {err_msg}") + powershell_error = ", ".join(f"{k}: {v}" for k, v in errors.items()) + except Exception as e: + usage_logger.error("Failed to fetch Shared Mailbox / Public Folder stats via PowerShell", exc_info=True) + powershell_error = str(e) + + data.update({ + "shared_mailboxes_count": shared_count, + "shared_mailboxes_total_bytes": shared_bytes, + "shared_mailboxes_total_formatted": format_bytes(shared_bytes) if shared_bytes is not None else "Error/Unavailable", + "public_folders_count": pf_count, + "public_folders_total_bytes": pf_bytes, + "public_folders_total_formatted": format_bytes(pf_bytes) if pf_bytes is not None else "Error/Unavailable", + "mail_public_folders_count": mail_pf_count, + "powershell_error": powershell_error + }) + + usage_logger.info("Mailbox Usage Telemetry Pipeline completed successfully.") + return data + +# ================================================================================= +# MODULAR UI COMPONENTS +# ================================================================================= + +class MailboxUsageFrame(ctk.CTkFrame): + """Self-contained customtkinter component wrapping Exchange Online Mailbox Usage UI.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None # 'loading', 'success', 'error', None + + self.build_ui() + + def build_ui(self): + """Creates card container for the tab.""" + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + ctk.CTkLabel(self.inner_pad, text="Exchange Online Mailbox Usage", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(anchor="w", pady=(0, 10)) + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.warning_label = ctk.CTkLabel(self.inner_pad, text="", font=FONT_BODY_MEDIUM, text_color=COLOR_ERROR, justify="left", anchor="w", wraplength=750) + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.reset_view() + + def reset_view(self): + """Resets and hides grids.""" + self.pack_forget() + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + if hasattr(self, "warning_label"): + self.warning_label.pack_forget() + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=20) + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + display_msg = "Reports telemetry permission required.\nPlease grant the 'Reports.Read.All' application permission to your App Registration in Microsoft Entra ID." + + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 10)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + """Triggers parallel fetches inside isolated background threads.""" + usage_logger.info("Mailbox Usage trigger_fetch called. Spawning background worker thread...") + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=10) + self.grid_frame.pack_forget() + + self._set_state_loading("Downloading and parsing Mailbox Usage reports...") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str): + usage_logger.info("Executing thread: _execute_mailbox_worker") + if self.semaphore: + self.semaphore.acquire() + try: + data = run_mailbox_usage_pipeline(client_id, client_secret, tenant) + usage_logger.info("Successfully completed Mailbox Usage telemetry data fetch.") + self.after(0, self._render_success, data) + except Exception as e: + usage_logger.error("Exception caught in Mailbox Usage worker.", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, data: dict): + self.last_data = data + usage_logger.info("Mailbox Usage data successfully retrieved. Rendering UI grid.") + self.state_frame.pack_forget() + for w in self.grid_frame.winfo_children(): + w.destroy() + + if data.get("powershell_error"): + err_msg = data["powershell_error"] + if "powershell" in err_msg.lower() or "pwsh" in err_msg.lower(): + friendly_msg = "PowerShell Core ('pwsh') not installed/configured. Cannot retrieve shared mailbox or public folder statistics." + elif "exchangeonlinemanagement" in err_msg.lower(): + friendly_msg = "ExchangeOnlineManagement PowerShell module is missing. Run: Install-Module -Name ExchangeOnlineManagement -Scope CurrentUser" + else: + friendly_msg = f"Failed to retrieve shared mailbox or public folder statistics: {err_msg}" + + self.warning_label.configure(text=f"⚠️ Warning: {friendly_msg}") + self.warning_label.pack(anchor="w", pady=(0, 10)) + else: + self.warning_label.pack_forget() + + self.grid_frame.pack(fill="x", expand=True) + + self.grid_frame.grid_columnconfigure(0, weight=3) + self.grid_frame.grid_columnconfigure(1, weight=2) + + headers_sp = ["Mailbox Metric Description", "Value / Measurement"] + for col_idx, head_text in enumerate(headers_sp): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + rows_data = [ + ("Total Mailboxes Analyzed", f"{data.get('total_mailboxes', 0):,} Mailboxes"), + ("Total Size of All Mailboxes", data.get("total_storage_formatted", "0.00 Bytes")), + ("Average Mailbox Size", data.get("average_mailbox_size_formatted", "0.00 Bytes")), + ("Total Number of Emails", f"{data.get('total_emails', 0):,} Emails"), + ("Average Emails per User", f"{data.get('average_emails', 0.0):,.0f} Emails") + ] + + # PowerShell retrieved stats (with individual error handling) + s_count = data.get("shared_mailboxes_count") + s_count_str = f"{s_count:,} Shared Mailboxes" if s_count is not None else "Error/Unavailable" + s_size_str = data.get("shared_mailboxes_total_formatted", "Error/Unavailable") + + pf_count = data.get("public_folders_count") + pf_count_str = f"{pf_count:,} Public Folders" if pf_count is not None else "Error/Unavailable" + + mail_pf_count = data.get("mail_public_folders_count") + mail_pf_count_str = f"{mail_pf_count:,} Public Folders" if mail_pf_count is not None else "Error/Unavailable" + + pf_size_str = data.get("public_folders_total_formatted", "Error/Unavailable") + + rows_data += [ + ("Shared Mailboxes Count", s_count_str), + ("Total Shared Mailbox Size", s_size_str), + ("Public Folders Count", pf_count_str), + ("Mail-enabled Public Folders Count", mail_pf_count_str), + ("Total Public Folder Size", pf_size_str) + ] + + for r_idx, (metric_name, val) in enumerate(rows_data, start=1): + bg_style = "transparent" if r_idx % 2 != 0 else COLOR_SURFACE_VARIANT + + c0 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c0.grid(row=r_idx, column=0, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(c0, text=metric_name, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + c1 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c1.grid(row=r_idx, column=1, sticky="nsew", padx=1, pady=1) + ctk.CTkLabel(c1, text=val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + self.status = "success" + self.on_status_change() + + def _render_error(self, err_msg): + usage_logger.warning(f"Mailbox Usage fetch failed: {err_msg}") + self._set_state_error(err_msg) + self.status = "error" + self.on_status_change() diff --git a/telemetry/pdf_report.py b/telemetry/pdf_report.py new file mode 100644 index 00000000..6e9391ee --- /dev/null +++ b/telemetry/pdf_report.py @@ -0,0 +1,1001 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""PDF Report Compilation module for Microsoft 365 Tenant Telemetry data.""" + +import io +from datetime import datetime +from reportlab.lib.pagesizes import letter +from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Table, TableStyle, Image, PageBreak, KeepTogether +from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle +from reportlab.lib import colors +from reportlab.pdfgen import canvas + + +class NumberedCanvas(canvas.Canvas): + """Custom canvas to compute total page count and draw running headers, footers and page numbers.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._saved_page_states = [] + + def showPage(self): + self._saved_page_states.append(dict(self.__dict__)) + self._startPage() + + def save(self): + num_pages = len(self._saved_page_states) + for state in self._saved_page_states: + self.__dict__.update(state) + self.draw_page_decorations(num_pages) + super().showPage() + super().save() + + def draw_page_decorations(self, page_count): + if self._pageNumber == 1: + # Skip page number and decorations on the cover page + return + + self.saveState() + self.setFont("Helvetica-Bold", 8) + self.setFillColor(colors.HexColor("#1E3A8A")) + + # Header + self.drawString(54, 750, "DEAL ASSISTANT") + self.setFont("Helvetica", 8) + self.setFillColor(colors.HexColor("#64748B")) + self.drawRightString(558, 750, "M365 Tenant Telemetry & Audit Report") + + # Header Line + self.setStrokeColor(colors.HexColor("#E2E8F0")) + self.setLineWidth(0.5) + self.line(54, 742, 558, 742) + + # Footer Line + self.line(54, 52, 558, 52) + + # Footer + self.drawString(54, 40, "Confidential - Tenant Audit Assessment") + page_text = f"Page {self._pageNumber} of {page_count}" + self.drawRightString(558, 40, page_text) + + self.restoreState() + + +def format_prepaid_units(item: dict) -> str: + prepaid = item.get("prepaidUnits", {}) + p_str = f"Enabled: {prepaid.get('enabled', 0):,}" + if prepaid.get('warning', 0) > 0: + p_str += f"\nWarn: {prepaid.get('warning'):,}" + if prepaid.get('suspended', 0) > 0: + p_str += f"\nSusp: {prepaid.get('suspended'):,}" + return p_str + + +def generate_trend_chart_bytes(trend_data: dict) -> io.BytesIO: + """Generates the O365 Active User Trend Chart on-the-fly to minimize persistent memory footprint.""" + from matplotlib.figure import Figure + from matplotlib.backends.backend_agg import FigureCanvasAgg + + dates = trend_data.get("dates", []) + if not dates: + return None + + fig = Figure(figsize=(6.5, 3.2), dpi=150) + ax = fig.add_subplot(111) + fig.patch.set_facecolor("#FFFFFF") + ax.set_facecolor("#FFFFFF") + + # Palette tailored to Match Deal Assistant theme + ax.plot(dates, trend_data.get("office365", []), marker='o', markersize=3, linewidth=1.5, label='Office 365', color="#1E3A8A") + ax.plot(dates, trend_data.get("exchange", []), marker='o', markersize=3, linewidth=1.5, label='Exchange', color="#C2410C") + ax.plot(dates, trend_data.get("onedrive", []), marker='o', markersize=3, linewidth=1.5, label='OneDrive', color="#3B82F6") + ax.plot(dates, trend_data.get("sharepoint", []), marker='o', markersize=3, linewidth=1.5, label='SharePoint', color="#15803D") + ax.plot(dates, trend_data.get("teams", []), marker='o', markersize=3, linewidth=1.5, label='Teams', color="#9333EA") + + ax.set_xlabel("Date", fontsize=8, color="#475569") + ax.set_ylabel("Active Users", fontsize=8, color="#475569") + ax.tick_params(axis='x', colors="#475569", rotation=45, labelsize=7) + ax.tick_params(axis='y', colors="#475569", labelsize=7) + + if len(dates) > 10: + ax.set_xticks(dates[::max(1, len(dates)//10)]) + + for spine in ax.spines.values(): + spine.set_color("#CBD5E1") + + ax.legend(facecolor="#FFFFFF", edgecolor="#CBD5E1", labelcolor="#1E293B", fontsize=8) + fig.tight_layout() + + buf = io.BytesIO() + canvas = FigureCanvasAgg(fig) + canvas.print_png(buf) + buf.seek(0) + return buf + + +def generate_pa_chart_bytes(pa: dict) -> io.BytesIO: + """Generates the Power Automate Flows breakdown bar chart on-the-fly.""" + from matplotlib.figure import Figure + from matplotlib.backends.backend_agg import FigureCanvasAgg + + counts = pa.get("counts", {}) + if not counts: + return None + + active_counts = pa.get("active_counts", {}) + tier_counts = pa.get("tier_counts", {}) + active_tier_counts = pa.get("active_tier_counts", {}) + complex_flows = pa.get("complex_logic_flows", []) + + fig = Figure(figsize=(6.5, 3.2), dpi=150) + ax = fig.add_subplot(111) + fig.patch.set_facecolor("#FFFFFF") + ax.set_facecolor("#FFFFFF") + + categories = ['Cloud Flows', 'Desktop Flows', 'Personal', 'Enterprise', 'Complex'] + + c_total = counts.get("Cloud Flows", 0) + c_active = active_counts.get("Cloud Flows", 0) + c_inactive = c_total - c_active + + d_total = counts.get("Desktop Flows", 0) + d_active = active_counts.get("Desktop Flows", 0) + d_inactive = d_total - d_active + + p_total = tier_counts.get("Personal Productivity", 0) + p_active = active_tier_counts.get("Personal Productivity", 0) + p_inactive = p_total - p_active + + e_total = tier_counts.get("Enterprise/Departmental", 0) + e_active = active_tier_counts.get("Enterprise/Departmental", 0) + e_inactive = e_total - e_active + + complex_active = sum(1 for f in complex_flows if f.get("Active") == "Yes") + complex_inactive = len(complex_flows) - complex_active + + actives = [c_active, d_active, p_active, e_active, complex_active] + inactives = [c_inactive, d_inactive, p_inactive, e_inactive, complex_inactive] + + x = range(len(categories)) + width = 0.25 + + rects1 = ax.bar(x, actives, width, label='Active', color="#1E3A8A") + rects2 = ax.bar([i + width for i in x], inactives, width, label='Inactive', color="#CBD5E1") + + ax.set_ylabel('Count', color="#1E293B", fontsize=8, fontweight='bold') + ax.set_title('Power Automate Flows Breakdown', color="#1E293B", fontsize=9, fontweight='bold') + ax.set_xticks([i + width/2 for i in x]) + ax.set_xticklabels(categories, color="#1E293B", fontsize=8, fontweight='bold') + ax.legend(facecolor="#FFFFFF", edgecolor="#CBD5E1", labelcolor="#1E293B", prop={'size':8}) + + ax.bar_label(rects1, padding=2, color="#1E293B", fontsize=7) + ax.bar_label(rects2, padding=2, color="#1E293B", fontsize=7) + + for spine in ax.spines.values(): + spine.set_color("#CBD5E1") + + ax.tick_params(axis='y', colors="#1E293B", labelsize=8) + + max_val = max(max(actives), max(inactives)) + ax.set_ylim(0, max(max_val + 3, int(max_val * 1.3))) + + fig.tight_layout() + buf = io.BytesIO() + canvas = FigureCanvasAgg(fig) + canvas.print_png(buf) + buf.seek(0) + return buf + + +def generate_pdf_report(data: dict, filepath: str): + """Generates a beautifully structured PDF document summarizing all tenant telemetry statistics.""" + + # 1. Document Setup + # 54pt margins correspond to 0.75 inches + doc = SimpleDocTemplate( + filepath, + pagesize=letter, + leftMargin=54, + rightMargin=54, + topMargin=64, + bottomMargin=64 + ) + + styles = getSampleStyleSheet() + + # Custom color palette + primary_color = colors.HexColor("#1E3A8A") # Navy Accent + secondary_color = colors.HexColor("#475569") # Slate Secondary + text_color = colors.HexColor("#1E293B") # Charcoal Body Text + outline_color = colors.HexColor("#CBD5E1") # Border light grey + + # Modify default styles in-place + styles['Normal'].textColor = text_color + styles['Normal'].fontSize = 9 + styles['Normal'].leading = 13 + + # Custom styles + title_style = ParagraphStyle( + 'CoverTitle', + parent=styles['Normal'], + fontName='Helvetica-Bold', + fontSize=26, + leading=32, + textColor=primary_color, + spaceAfter=10 + ) + + subtitle_style = ParagraphStyle( + 'CoverSubtitle', + parent=styles['Normal'], + fontName='Helvetica', + fontSize=13, + leading=18, + textColor=secondary_color, + spaceAfter=30 + ) + + h1_style = ParagraphStyle( + 'SectionH1', + parent=styles['Normal'], + fontName='Helvetica-Bold', + fontSize=15, + leading=18, + textColor=primary_color, + spaceBefore=22, + spaceAfter=10, + keepWithNext=True + ) + + h2_style = ParagraphStyle( + 'SectionH2', + parent=styles['Normal'], + fontName='Helvetica-Bold', + fontSize=11, + leading=14, + textColor=secondary_color, + spaceBefore=14, + spaceAfter=6, + keepWithNext=True + ) + + body_style = ParagraphStyle( + 'ReportBody', + parent=styles['Normal'], + fontSize=9, + leading=13, + spaceAfter=6 + ) + + bold_body_style = ParagraphStyle( + 'ReportBodyBold', + parent=body_style, + fontName='Helvetica-Bold' + ) + + table_cell_style = ParagraphStyle( + 'TableCell', + parent=styles['Normal'], + fontSize=8.5, + leading=11 + ) + + table_cell_bold = ParagraphStyle( + 'TableCellBold', + parent=table_cell_style, + fontName='Helvetica-Bold', + textColor=primary_color + ) + + table_cell_header = ParagraphStyle( + 'TableCellHeader', + parent=table_cell_style, + fontName='Helvetica-Bold', + textColor=colors.white + ) + + meta_label_style = ParagraphStyle( + 'MetaLabel', + parent=styles['Normal'], + fontName='Helvetica-Bold', + fontSize=10, + textColor=secondary_color + ) + + meta_val_style = ParagraphStyle( + 'MetaValue', + parent=styles['Normal'], + fontSize=10, + textColor=text_color + ) + + story = [] + + # ========================================================================= + # COVER PAGE + # ========================================================================= + story.append(Spacer(1, 120)) + story.append(Paragraph("🤝 Deal Assistant", ParagraphStyle('Branding', parent=styles['Normal'], fontName='Helvetica-Bold', fontSize=18, textColor=primary_color, spaceAfter=20))) + story.append(Paragraph("Microsoft 365 Tenant
Audit & Telemetry Report", title_style)) + story.append(Paragraph("A comprehensive assessment of license allocations, workload adoption patterns, security configurations, and workflow automation.", subtitle_style)) + story.append(Spacer(1, 100)) + + # Metadata Table + meta_data = [ + [Paragraph("Tenant Name / ID:", meta_label_style), Paragraph(data.get("tenant_id", "N/A"), meta_val_style)], + [Paragraph("Report Generated:", meta_label_style), Paragraph(datetime.now().strftime("%B %d, %Y at %I:%M %p"), meta_val_style)], + [Paragraph("Assessment Status:", meta_label_style), Paragraph("🟢 Audit Completed Successfully", ParagraphStyle('StatusStyle', parent=meta_val_style, fontName='Helvetica-Bold', textColor=colors.HexColor("#15803D")))], + [Paragraph("Report Context:", meta_label_style), Paragraph("Usage & Adoption Inventory", meta_val_style)] + ] + + meta_table = Table(meta_data, colWidths=[130, 370]) + meta_table.setStyle(TableStyle([ + ('VALIGN', (0, 0), (-1, -1), 'TOP'), + ('BOTTOMPADDING', (0, 0), (-1, -1), 8), + ('LINEBELOW', (0, 0), (-1, -2), 0.5, colors.HexColor("#F1F5F9")), + ])) + story.append(meta_table) + story.append(PageBreak()) + + # ========================================================================= + # SECTION 1: SUBSCRIBED SKUS INVENTORY + # ========================================================================= + story.append(Paragraph("1. Subscribed SKUs", h1_style)) + story.append(Paragraph("This section outlines the licensing packages (SKUs) currently configured and active in your Microsoft Entra ID tenant scope, displaying total enabled vs. consumed license counts.", body_style)) + story.append(Spacer(1, 8)) + + sku_list = data.get("skus", []) + if not sku_list: + story.append(Paragraph("No subscribed licensing data was discovered or available for this report.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + # Table columns: SKU, Units, Consumed + sku_table_data = [[ + Paragraph("SKU Part Number", table_cell_header), + Paragraph("Allocated Units Status", table_cell_header), + Paragraph("Consumed Units", table_cell_header) + ]] + + for item in sku_list: + sku_table_data.append([ + Paragraph(item.get("skuPartNumber", "UNKNOWN_SKU"), table_cell_bold), + Paragraph(format_prepaid_units(item).replace("\n", "
"), table_cell_style), + Paragraph(f"{item.get('consumedUnits', 0):,}", table_cell_style) + ]) + + sku_table = Table(sku_table_data, colWidths=[220, 160, 120]) + sku_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 6), + ('BOTTOMPADDING', (0, 0), (-1, -1), 6), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(sku_table) + story.append(Spacer(1, 15)) + + # ========================================================================= + # SECTION 1b: DIRECTORY SUMMARY + # ========================================================================= + story.append(Paragraph("1b. Directory Summary", h1_style)) + + dir_data = data.get("directory", {}) + if not dir_data: + story.append(Paragraph("No directory telemetry data was available.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + # 1. Domains Table + story.append(Paragraph("Domains", h2_style)) + story.append(Paragraph("This section displays the configured internet domains associated with the tenant and their verified statuses.", body_style)) + story.append(Spacer(1, 8)) + + domains = dir_data.get("domains", []) + if not domains: + story.append(Paragraph("No domains discovered in directory scope.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + domains_table_data = [[ + Paragraph("Domain ID", table_cell_header), + Paragraph("Admin Managed", table_cell_header), + Paragraph("Default", table_cell_header), + Paragraph("Verified", table_cell_header), + Paragraph("Supported Services", table_cell_header) + ]] + for item in domains: + admin_managed = "Yes" if item.get("isAdminManaged") else "No" + is_default = "Yes" if item.get("isDefault") else "No" + is_verified = "Yes" if item.get("isVerified") else "No" + services = item.get("supportedServices", []) + services_str = ", ".join(services) if services else "-" + + domains_table_data.append([ + Paragraph(item.get("id", "-"), table_cell_bold), + Paragraph(admin_managed, table_cell_style), + Paragraph(is_default, table_cell_style), + Paragraph(is_verified, table_cell_style), + Paragraph(services_str, table_cell_style) + ]) + domains_table = Table(domains_table_data, colWidths=[140, 85, 55, 55, 165]) + domains_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(domains_table) + + story.append(Spacer(1, 15)) + + # 2. Groups & Users Table + story.append(Paragraph("Groups & Users", h2_style)) + story.append(Paragraph("This section displays counts of different directory user and group categories configured in Microsoft Entra ID.", body_style)) + story.append(Spacer(1, 8)) + + group_counts = dir_data.get("group_counts", {}) + user_counts = dir_data.get("user_counts", {}) + + dir_table_data = [[ + Paragraph("Category", table_cell_header), + Paragraph("Count", table_cell_header) + ]] + + rows_spec = [ + # User statistics + ("Total Users", user_counts.get("total", 0), True), + ("Enabled Users", user_counts.get("enabled", 0), False), + ("Disabled Users", user_counts.get("disabled", 0), False), + ("Member Users", user_counts.get("member", 0), False), + ("Guest Users", user_counts.get("guest", 0), False), + # Spacing placeholder + ("", "", False), + # Group statistics + ("Total Groups", group_counts.get("total", 0), True), + ("Microsoft 365 Groups (Unified)", group_counts.get("m365", 0), False), + ("Security Groups (Static, non-mail-enabled)", group_counts.get("security", 0), False), + ("Mail-enabled Security Groups", group_counts.get("mail_enabled_security", 0), False), + ("Distribution Groups", group_counts.get("distribution", 0), False), + ("Dynamic Groups (Dynamic Membership)", group_counts.get("dynamic", 0), False) + ] + + row_backgrounds = [] + for idx, item in enumerate(rows_spec, start=1): + metric_name, val, is_bold = item + if metric_name == "": + dir_table_data.append([Paragraph("", table_cell_style), Paragraph("", table_cell_style)]) + # Divider background color + row_backgrounds.append((idx, colors.HexColor("#CBD5E1"))) + continue + + cell_bold = table_cell_bold if is_bold else table_cell_style + dir_table_data.append([ + Paragraph(metric_name, cell_bold), + Paragraph(f"{val:,}", table_cell_style) + ]) + # Alternate row background + bg = colors.white if idx % 2 == 0 else colors.HexColor("#F8FAFC") + row_backgrounds.append((idx, bg)) + + dir_table_style = [ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ] + + for r_idx, bg_color in row_backgrounds: + dir_table_style.append(('BACKGROUND', (0, r_idx), (-1, r_idx), bg_color)) + + dir_table = Table(dir_table_data, colWidths=[300, 200]) + dir_table.setStyle(TableStyle(dir_table_style)) + story.append(dir_table) + story.append(Spacer(1, 15)) + + + # ========================================================================= + # SECTION 2: APP USAGE SUMMARY + # ========================================================================= + story.append(Paragraph("2. App Usage Summary", h1_style)) + story.append(Paragraph("Active Users Usage", h2_style)) + story.append(Paragraph("A breakdown of user activity across major Microsoft 365 services over the last 30, 90, and 180 days, representing actual adoption levels.", body_style)) + story.append(Spacer(1, 8)) + + o365_usage = data.get("o365_usage", []) + if not o365_usage: + story.append(Paragraph("No active user usage report data was available.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + usage_table_data = [[ + Paragraph("Service / License", table_cell_header), + Paragraph("30 Days Active", table_cell_header), + Paragraph("90 Days Active", table_cell_header), + Paragraph("180 Days Active", table_cell_header) + ]] + + for row in o365_usage: + usage_table_data.append([ + Paragraph(str(row[0]), table_cell_bold), + Paragraph(f"{row[1]:,}", table_cell_style), + Paragraph(f"{row[2]:,}", table_cell_style), + Paragraph(f"{row[3]:,}", table_cell_style) + ]) + + usage_table = Table(usage_table_data, colWidths=[200, 100, 100, 100]) + usage_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 6), + ('BOTTOMPADDING', (0, 0), (-1, -1), 6), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(usage_table) + + # 30-Day Trend Chart - Generated on the fly + o365_trend = data.get("o365_trend", {}) + if o365_trend and o365_trend.get("dates"): + try: + chart_bytes = generate_trend_chart_bytes(o365_trend) + if chart_bytes: + story.append(Spacer(1, 15)) + story.append(Paragraph("O365 30-Day Active User Trend", h2_style)) + chart_flow = Image(chart_bytes, width=450, height=210) + story.append(chart_flow) + except Exception as chart_ex: + print(f"Failed to generate active user trend chart for PDF: {chart_ex}") + + story.append(PageBreak()) + + # M365 Apps Usage + story.append(Paragraph("Microsoft 365 Client Applications Usage (180 Days)", h2_style)) + story.append(Paragraph("Displays the unique counts of active users on client applications (Outlook, Word, Excel, PowerPoint, OneNote, Teams) segmented by system platforms.", body_style)) + story.append(Spacer(1, 8)) + + m365_apps = data.get("m365_apps", []) + if not m365_apps: + story.append(Paragraph("No client application telemetry data was available.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + # Format 4-columns layout matching the UI table + app_table_data = [[ + Paragraph("App / Platform", table_cell_header), + Paragraph("Active Users", table_cell_header), + Paragraph("App / Platform", table_cell_header), + Paragraph("Active Users", table_cell_header) + ]] + + half = (len(m365_apps) + 1) // 2 + left_col = m365_apps[:half] + right_col = m365_apps[half:] + + for r_idx in range(half): + l_name = left_col[r_idx][0] if r_idx < len(left_col) else "" + l_val = f"{left_col[r_idx][1]:,}" if r_idx < len(left_col) else "" + r_name = right_col[r_idx][0] if r_idx < len(right_col) else "" + r_val = f"{right_col[r_idx][1]:,}" if r_idx < len(right_col) else "" + + app_table_data.append([ + Paragraph(l_name, table_cell_bold if l_name else table_cell_style), + Paragraph(l_val, table_cell_style), + Paragraph(r_name, table_cell_bold if r_name else table_cell_style), + Paragraph(r_val, table_cell_style) + ]) + + app_table = Table(app_table_data, colWidths=[150, 100, 150, 100]) + app_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(app_table) + + story.append(Spacer(1, 15)) + + # ========================================================================= + # SECTION 3: WORKLOAD STORAGE & METRICS + # ========================================================================= + story.append(Paragraph("3. Workload Storage & Environmental Telemetry", h1_style)) + story.append(Paragraph("A compiled summary of storage consumption, item counts, and device statistics across Exchange Online, SharePoint, and OneDrive workloads.", body_style)) + story.append(Spacer(1, 8)) + + # 3.1 Exchange Mailbox & Calendar Telemetry + story.append(Paragraph("Exchange Online Mailbox & Resource Configurations", h2_style)) + + mailbox = data.get("mailbox", {}) + calendar = data.get("calendar", {}) + + # Let's check for warning/errors + pw_warn = [] + if mailbox.get("powershell_error"): + pw_warn.append(f"Mailbox: {mailbox['powershell_error']}") + if calendar.get("powershell_error"): + pw_warn.append(f"Calendar: {calendar['powershell_error']}") + + if pw_warn: + story.append(Paragraph(f"⚠️ Warning: PowerShell metrics are restricted or incomplete ({'; '.join(pw_warn)})", ParagraphStyle('WarnTxt', parent=body_style, textColor=colors.HexColor("#D97706"), fontName="Helvetica-Bold"))) + story.append(Spacer(1, 4)) + + workload_table_data = [[ + Paragraph("Metric / Telemetry Property", table_cell_header), + Paragraph("Exchange Mailbox Value", table_cell_header) + ]] + + # Compile rows from mailbox & calendar + exchange_rows = [ + ("Total Mailboxes Analyzed", f"{mailbox.get('total_mailboxes', 0):,} Mailboxes"), + ("Total Size of All Mailboxes", mailbox.get("total_storage_formatted", "0.00 Bytes")), + ("Average Mailbox Size", mailbox.get("average_mailbox_size_formatted", "0.00 Bytes")), + ("Total Emails Volume", f"{mailbox.get('total_emails', 0):,} Emails"), + ("Average Emails per Mailbox", f"{mailbox.get('average_emails', 0.0):,.0f} Emails"), + ] + + s_count = mailbox.get('shared_mailboxes_count') + s_count_str = f"{s_count:,} Shared Mailboxes" if s_count is not None else "Error/Unavailable" + s_size_str = mailbox.get("shared_mailboxes_total_formatted", "Error/Unavailable") + + pf_count = mailbox.get('public_folders_count') + pf_count_str = f"{pf_count:,} Public Folders" if pf_count is not None else "Error/Unavailable" + + mail_pf_count = mailbox.get('mail_public_folders_count') + mail_pf_count_str = f"{mail_pf_count:,} Public Folders" if mail_pf_count is not None else "Error/Unavailable" + + pf_size_str = mailbox.get("public_folders_total_formatted", "Error/Unavailable") + + exchange_rows += [ + ("Shared Mailboxes Count", s_count_str), + ("Total Shared Mailbox Size", s_size_str), + ("Public Folders Count", pf_count_str), + ("Mail-enabled Public Folders Count", mail_pf_count_str), + ("Total Public Folder Size", pf_size_str), + ] + + # Add calendar properties + reserve_val = calendar.get("CanUsersReserveRooms") + if isinstance(reserve_val, bool): reserve_val = "Yes" if reserve_val else "No" + + att_val = calendar.get("CanShareAttachments") + if isinstance(att_val, bool): att_val = "Yes" if att_val else "No" + + exchange_rows += [ + ("Room & Resource Reservation Enabled", str(reserve_val)), + ("Calendar Resource Pools (Rooms/Devices)", calendar.get("NamingConvention") or "None found"), + ("Calendar Attachment Link Permissions", str(att_val)), + ] + + for label, val in exchange_rows: + workload_table_data.append([ + Paragraph(label, table_cell_bold), + Paragraph(val, table_cell_style) + ]) + + ex_table = Table(workload_table_data, colWidths=[260, 240]) + ex_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(ex_table) + story.append(Spacer(1, 15)) + + # 3.1b Integrated Apps + story.append(Paragraph("Integrated Apps", h2_style)) + story.append(Paragraph("This section lists all organization-wide apps deployed in Exchange Online by administrators and their enabled status.", body_style)) + story.append(Spacer(1, 8)) + + org_apps = calendar.get("OrganizationApps", []) + apps_error = calendar.get("AppsError") + + if apps_error: + story.append(Paragraph(f"Error querying organization apps: {apps_error}", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + elif not org_apps: + story.append(Paragraph("No organization-wide apps found in Exchange Online.", body_style)) + else: + apps_table_data = [[ + Paragraph("App Display Name", table_cell_header), + Paragraph("Status", table_cell_header), + Paragraph("App Display Name", table_cell_header), + Paragraph("Status", table_cell_header) + ]] + half = (len(org_apps) + 1) // 2 + left_col = org_apps[:half] + right_col = org_apps[half:] + + for r_idx in range(half): + row_items = [] + if r_idx < len(left_col): + app = left_col[r_idx] + enabled_str = "Enabled" if app.get("Enabled") else "Disabled" + row_items.extend([app.get("DisplayName", "-"), enabled_str]) + else: + row_items.extend(["", ""]) + + if r_idx < len(right_col): + app = right_col[r_idx] + enabled_str = "Enabled" if app.get("Enabled") else "Disabled" + row_items.extend([app.get("DisplayName", "-"), enabled_str]) + else: + row_items.extend(["", ""]) + + apps_table_data.append([ + Paragraph(row_items[0], table_cell_bold if row_items[0] else table_cell_style), + Paragraph(row_items[1], table_cell_style), + Paragraph(row_items[2], table_cell_bold if row_items[2] else table_cell_style), + Paragraph(row_items[3], table_cell_style) + ]) + + apps_table = Table(apps_table_data, colWidths=[180, 70, 180, 70]) + apps_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(apps_table) + + story.append(Spacer(1, 15)) + story.append(PageBreak()) + + # 3.2 SharePoint & OneDrive Storage + story.append(Paragraph("SharePoint & OneDrive Environment Telemetry", h2_style)) + story.append(Paragraph("A comparison of file volume, storage consumption, site activity, and active synchronization clients.", body_style)) + story.append(Spacer(1, 8)) + + sp = data.get("sharepoint", {}) + od = data.get("onedrive", {}) + + files_table_data = [[ + Paragraph("Metric Property Description", table_cell_header), + Paragraph("SharePoint Sites (180d)", table_cell_header), + Paragraph("OneDrive Personal (180d)", table_cell_header) + ]] + + files_rows = [ + ("Total Scope Count (Sites / Accounts)", f"{sp.get('total_sites', 0):,} Sites", f"{od.get('total_accounts', 0):,} Accounts"), + ("Total Storage Consumed", sp.get("total_storage_formatted", "0.00 Bytes"), od.get("total_storage_formatted", "0.00 Bytes")), + ("Total Stored File Count", f"{sp.get('total_files', 0):,} Files", f"{od.get('total_files', 0):,} Files"), + ("Active Files Count (Active %)", f"{sp.get('active_files', 0):,} ({sp.get('active_files_pct', 0.0):.1f}%)", f"{od.get('active_files', 0):,} ({od.get('active_files_pct', 0.0):.1f}%)"), + ("Users with Sync Client Active", "N/A (SharePoint level)", f"{od.get('sync_users', 0):,} Users ({od.get('sync_users_pct', 0.0):.1f}%)"), + ("Active OneNote Users", "N/A (SharePoint level)", f"{od.get('onenote_users', 0):,} Users"), + ] + + for label, sp_val, od_val in files_rows: + files_table_data.append([ + Paragraph(label, table_cell_bold), + Paragraph(sp_val, table_cell_style), + Paragraph(od_val, table_cell_style) + ]) + + files_table = Table(files_table_data, colWidths=[200, 150, 150]) + files_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 6), + ('BOTTOMPADDING', (0, 0), (-1, -1), 6), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(files_table) + story.append(Spacer(1, 15)) + + # ========================================================================= + # SECTION 4: DATA SECURITY & GOVERNANCE + # ========================================================================= + story.append(Paragraph("4. Data Security, Governance & Compliance", h1_style)) + story.append(Paragraph("A summary of classification sensitivity labels and data retention lifecycle policies configured within Microsoft Purview to protect corporate properties.", body_style)) + + # 4.1 Sensitivity Labels + story.append(Paragraph("Microsoft Purview Sensitivity Labels", h2_style)) + labels = data.get("security_labels", []) + if not labels: + story.append(Paragraph("No Purview Sensitivity Labels configured or permission restricted.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + labels_table_data = [[ + Paragraph("Sensitivity Label", table_cell_header), + Paragraph("Description", table_cell_header), + Paragraph("Shield", table_cell_header), + Paragraph("Mode", table_cell_header), + Paragraph("Priority", table_cell_header), + Paragraph("Status", table_cell_header) + ]] + + # Flatten parent labels and sublabels for the PDF table + flattened_labels = [] + for parent in labels: + flattened_labels.append({ + "name": parent.get("name", "N/A"), + "description": parent.get("description", "") or parent.get("toolTip", "") or "N/A", + "hasProtection": parent.get("hasProtection", False), + "applicationMode": parent.get("applicationMode", "N/A") or "N/A", + "priority": parent.get("priority", 0), + "isEnabled": parent.get("isEnabled", True), + "is_sub": False + }) + for sub in parent.get("sublabels", []): + flattened_labels.append({ + "name": f" L_ {sub.get('name', 'N/A')}", + "description": sub.get("description", "") or sub.get("toolTip", "") or "N/A", + "hasProtection": sub.get("hasProtection", False), + "applicationMode": sub.get("applicationMode", "N/A") or "N/A", + "priority": sub.get("priority", 0), + "isEnabled": sub.get("isEnabled", True), + "is_sub": True + }) + + for item in flattened_labels: + bg_bold_s = table_cell_bold if not item["is_sub"] else table_cell_style + protection_str = "Yes" if item["hasProtection"] else "No" + status_str = "Enabled" if item["isEnabled"] else "Disabled" + + labels_table_data.append([ + Paragraph(item["name"], bg_bold_s), + Paragraph(item["description"], table_cell_style), + Paragraph(protection_str, table_cell_style), + Paragraph(str(item["applicationMode"]).capitalize(), table_cell_style), + Paragraph(str(item["priority"]), table_cell_style), + Paragraph(status_str, table_cell_style) + ]) + + labels_table = Table(labels_table_data, colWidths=[120, 160, 50, 60, 50, 60]) + labels_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(labels_table) + + story.append(PageBreak()) + + # 4.2 Retention Policies + story.append(Paragraph("Microsoft Purview Retention Compliance Policies", h2_style)) + policies = data.get("retention_policies", []) + if not policies: + story.append(Paragraph("No Purview Retention compliance policies discovered or permission restricted.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + ret_table_data = [[ + Paragraph("Policy Name", table_cell_header), + Paragraph("Workloads Involved", table_cell_header), + Paragraph("Retention Duration Basis", table_cell_header), + Paragraph("Distribution Status", table_cell_header), + Paragraph("Status", table_cell_header) + ]] + + policies_list = policies if isinstance(policies, list) else [policies] + for policy in policies_list: + duration_val = str(policy.get("Duration", "N/A")) + duration_str = duration_val + if duration_val.lower() == "unlimited": + duration_str = "Keep Forever" + elif duration_val.isdigit(): + days = int(duration_val) + if days >= 365: + years = days / 365.0 + duration_str = f"{int(years)} Years ({days} days)" if years.is_integer() else f"{years:.1f} Years ({days} days)" + else: + duration_str = f"{days} days" + + trigger_val = policy.get("RetentionTrigger", "N/A") + if trigger_val and trigger_val != "N/A": + trigger_map = {"DateCreated": "created date", "DateModified": "last modified date", "DateLabeled": "labeled date"} + duration_str += f"
(from {trigger_map.get(trigger_val, trigger_val)})" + + enabled_val = policy.get("Enabled", True) + is_enabled = enabled_val.lower() == "true" if isinstance(enabled_val, str) else bool(enabled_val) + status_str = "Enabled" if is_enabled else "Disabled" + + ret_table_data.append([ + Paragraph(policy.get("Name", "N/A"), table_cell_bold), + Paragraph(policy.get("Workload", "N/A"), table_cell_style), + Paragraph(duration_str, table_cell_style), + Paragraph(policy.get("DistributionStatus", "Success"), table_cell_style), + Paragraph(status_str, table_cell_style) + ]) + + ret_table = Table(ret_table_data, colWidths=[130, 110, 110, 90, 60]) + ret_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(ret_table) + story.append(Spacer(1, 15)) + + # ========================================================================= + # SECTION 5: POWER AUTOMATE + # ========================================================================= + story.append(Paragraph("5. Power Platform & Automate Flows Analytics", h1_style)) + story.append(Paragraph("An analysis of low-code cloud and desktop workflows configured inside the tenant environments, identifying complex workflows and premium connectors.", body_style)) + story.append(Spacer(1, 8)) + + pa = data.get("power_automate", {}) + if not pa: + story.append(Paragraph("No Power Platform or Power Automate telemetry scan data was available.", ParagraphStyle('ErrTxt', parent=body_style, textColor=colors.HexColor("#DC2626")))) + else: + counts = pa.get("counts", {}) + total_flows = counts.get("Cloud Flows", 0) + counts.get("Desktop Flows", 0) + premium_conns = pa.get("premium_connectors", []) + custom_conns = pa.get("custom_connectors", []) + + prem_str = ", ".join(premium_conns) if premium_conns else "0" + cust_str = ", ".join(custom_conns) if custom_conns else "0" + + pa_table_data = [[ + Paragraph("Power Platform Telemetry Property", table_cell_header), + Paragraph("Scanned Value", table_cell_header) + ]] + + pa_rows = [ + ("Total Environments Scanned", str(pa.get("total_environments", 0))), + ("Total Flows (Active + Inactive)", f"{total_flows:,} Flows"), + ("Active Cloud Flows Count", f"{pa.get('active_counts', {}).get('Cloud Flows', 0):,} Cloud Flows"), + ("Active Desktop Flows Count", f"{pa.get('active_counts', {}).get('Desktop Flows', 0):,} Desktop Flows"), + ("Premium Connectors In Use", prem_str), + ("Custom Connectors In Use", cust_str), + ("Complex Business-Logic Flows Identified", f"{len(pa.get('complex_logic_flows', [])):,} Flows"), + ] + + for label, val in pa_rows: + pa_table_data.append([ + Paragraph(label, table_cell_bold), + Paragraph(val, table_cell_style) + ]) + + pa_table = Table(pa_table_data, colWidths=[220, 280]) + pa_table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), primary_color), + ('ALIGN', (0, 0), (-1, -1), 'LEFT'), + ('VALIGN', (0, 0), (-1, -1), 'MIDDLE'), + ('TOPPADDING', (0, 0), (-1, -1), 5), + ('BOTTOMPADDING', (0, 0), (-1, -1), 5), + ('ROWBACKGROUNDS', (0, 1), (-1, -1), [colors.white, colors.HexColor("#F8FAFC")]), + ('GRID', (0, 0), (-1, -1), 0.5, outline_color), + ])) + story.append(pa_table) + + # Power Automate Breakdown Chart - Generated on the fly + if counts: + try: + pa_chart_bytes = generate_pa_chart_bytes(pa) + if pa_chart_bytes: + story.append(Spacer(1, 15)) + story.append(Paragraph("Power Automate Flows Breakdown Chart", h2_style)) + pa_chart = Image(pa_chart_bytes, width=450, height=210) + story.append(pa_chart) + except Exception as chart_ex: + print(f"Failed to generate Power Automate chart for PDF: {chart_ex}") + + # 4. Build Document + doc.build(story, canvasmaker=NumberedCanvas) diff --git a/telemetry/power_automate.py b/telemetry/power_automate.py new file mode 100644 index 00000000..363fa21d --- /dev/null +++ b/telemetry/power_automate.py @@ -0,0 +1,682 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Modular Power Automate telemetry scanner, analysis pipelines, and visual interfaces.""" + +import os +import time +import json +import logging +import threading +import requests +import pandas as pd +from datetime import datetime +from concurrent.futures import ThreadPoolExecutor, as_completed +from tkinter import filedialog, messagebox +from typing import Any, Dict, List +import customtkinter as ctk + +# Safely import matplotlib to embed plots in Tkinter +try: + from matplotlib.figure import Figure + from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg + MATPLOTLIB_AVAILABLE = True +except ImportError: + MATPLOTLIB_AVAILABLE = False + +# Import shared styles +from telemetry.styles import * + +# Bind to the async logger initialized in m365_telemetry.py +usage_logger = logging.getLogger("M365TelemetryAsyncLogger") + + +# ================================================================================= +# SCANNER LOGIC / PIPELINE +# ================================================================================= + +class PowerAutomateScanner: + def __init__(self, tenant_id, client_id, client_secret): + self.tenant_id = tenant_id + self.client_id = client_id + self.client_secret = client_secret + + self.log_dir = os.path.join("telemetry", "logs") + if not os.path.exists(self.log_dir): + os.makedirs(self.log_dir) + + self.log_file = os.path.join(self.log_dir, "power_automate_log.txt") + self._setup_logger() + self.access_token = None + + def _setup_logger(self): + """Configures logging to propagate to the central M365TelemetryAsyncLogger.""" + self.logger = logging.getLogger("M365TelemetryAsyncLogger.PowerAutomateScanner") + + def _get_access_token(self, scope): + """Fetches OAuth 2.0 token for a given scope.""" + self.logger.info(f"Step Start: Fetching access token for scope {scope} from Microsoft Identity Platform.") + url = f"https://login.microsoftonline.com/{self.tenant_id}/oauth2/v2.0/token" + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + payload = { + 'grant_type': 'client_credentials', + 'client_id': self.client_id, + 'client_secret': self.client_secret, + 'scope': scope + } + + try: + response = requests.post(url, headers=headers, data=payload) + response.raise_for_status() + token = response.json().get('access_token') + self.logger.info(f"Step End: Successfully retrieved access token for {scope}.") + return token + except Exception as e: + self.logger.error(f"Step Error: Failed to fetch access token for {scope}. Error: {str(e)}") + return None + + def fetch_all_pages(self, url, headers, context_name="API"): + """Helper to cleanly handle Power Platform API pagination & Throttling limits.""" + results = [] + while url: + res = requests.get(url, headers=headers) + + if res.status_code == 429: + retry_after = int(res.headers.get("Retry-After", 2)) + self.logger.warning(f"[!] Rate limited on {context_name}. Waiting {retry_after} seconds...") + time.sleep(retry_after) + continue + + if res.status_code == 200: + data = res.json() + results.extend(data.get("value", [])) + url = data.get("nextLink") or data.get("@odata.nextLink") + else: + self.logger.error(f"[X] {context_name} Request Failed | HTTP {res.status_code}: {res.text}") + break + + return results + + def fetch_single_resource(self, url, headers, context_name="API"): + """Helper to cleanly fetch a single resource handling Throttling limits for loop logic.""" + while True: + res = requests.get(url, headers=headers) + if res.status_code == 429: + retry_after = int(res.headers.get("Retry-After", 2)) + self.logger.warning(f"[!] Rate limited on {context_name}. Waiting {retry_after} seconds...") + time.sleep(retry_after) + continue + if res.status_code == 200: + return res.json() + self.logger.error(f"[X] {context_name} Request Failed | HTTP {res.status_code}: {res.text}") + return None + + def scan_flows(self): + """Scans Power Automate flows across all environments in the tenant.""" + self.logger.info("Main Process Start: Initiating Power Automate Flow Scan.") + + try: + bap_token = self._get_access_token("https://api.bap.microsoft.com/.default") + flow_token = self._get_access_token("https://service.flow.microsoft.com/.default") + except Exception as e: + self.logger.error(f"Auth Error: {e}") + return None + + if not bap_token or not flow_token: + self.logger.error("Main Process Failure: Aborting scan due to missing access tokens.") + return None + + bap_headers = {"Authorization": f"Bearer {bap_token}", "Accept": "application/json"} + flow_headers = {"Authorization": f"Bearer {flow_token}", "Accept": "application/json"} + + self.logger.info("Step Start: Fetching all environments in the tenant.") + env_api_url = "https://api.bap.microsoft.com/providers/Microsoft.BusinessAppPlatform/scopes/admin/environments?api-version=2023-06-01" + environments = self.fetch_all_pages(env_api_url, bap_headers, context_name="Environment Discovery") + + if not environments: + self.logger.error("[X] No environments found or failed to fetch.") + return None + + self.logger.info(f"[+] Successfully retrieved {len(environments)} environments.") + + counts = {"Cloud Flows": 0, "Desktop Flows": 0} + active_counts = {"Cloud Flows": 0, "Desktop Flows": 0} + tier_counts = {"Personal Productivity": 0, "Enterprise/Departmental": 0} + active_tier_counts = {"Personal Productivity": 0, "Enterprise/Departmental": 0} + premium_connectors_found = set() + custom_connectors_found = set() + import tempfile + import threading + cf_fd, complex_logic_flows_path = tempfile.mkstemp(suffix=".jsonl") + os.close(cf_fd) + complex_flows_lock = threading.Lock() + PREMIUM_KEYWORDS = ['shared_sql', 'shared_httpaction', 'shared_salesforce', 'shared_oracle', 'shared_sap'] + + for env in environments: + env_name = env.get("name") + env_props = env.get("properties", {}) + env_display = env_props.get("displayName", env_name) + is_default = env_props.get("isDefault", False) + + self.logger.info(f"[*] Scanning Environment: {env_display}") + + # ========================================== + # 1. FETCH CLOUD FLOWS + # ========================================== + flows_url = f"https://api.flow.microsoft.com/providers/Microsoft.ProcessSimple/scopes/admin/environments/{env_name}/v2/flows?api-version=2016-11-01" + cloud_flows = self.fetch_all_pages(flows_url, flow_headers, context_name="Cloud Flows Admin API (V2)") + + counts["Cloud Flows"] += len(cloud_flows) + active_cloud_flows = [f for f in cloud_flows if f.get("properties", {}).get("state") == "Started"] + active_counts["Cloud Flows"] += len(active_cloud_flows) + + self.logger.info(f" -> Cloud Flows: {len(cloud_flows)} total found, {len(active_cloud_flows)} are currently active.") + + # Fetch details in parallel using ThreadPoolExecutor + + def fetch_and_process_flow_detail(flow_summary): + flow_id = flow_summary.get("name") + state = flow_summary.get("properties", {}).get("state") + is_active = (state == "Started") + + detail_url = f"https://api.flow.microsoft.com/providers/Microsoft.ProcessSimple/scopes/admin/environments/{env_name}/flows/{flow_id}?api-version=2016-11-01" + flow_detail = self.fetch_single_resource(detail_url, flow_headers, context_name=f"Get Flow Details ({flow_id})") + if not flow_detail: + return None + return (flow_summary, flow_detail, is_active) + + with ThreadPoolExecutor(max_workers=15) as executor: + futures = {executor.submit(fetch_and_process_flow_detail, f): f for f in cloud_flows} + for future in as_completed(futures): + res = future.result() + if not res: + continue + + flow_summary, flow_detail, is_active = res + props = flow_detail.get("properties", {}) + name = props.get("displayName", "Unnamed Flow") + + is_managed = "workflowEntityId" in props or not is_default + if is_managed: + tier_counts["Enterprise/Departmental"] += 1 + tier = "Enterprise" + if is_active: + active_tier_counts["Enterprise/Departmental"] += 1 + else: + tier_counts["Personal Productivity"] += 1 + tier = "Personal" + if is_active: + active_tier_counts["Personal Productivity"] += 1 + + conn_refs = props.get("connectionReferences", {}) + for conn_key, conn_val in conn_refs.items(): + api_obj = conn_val.get("api", {}) + api_id = api_obj.get("id", "") + conn_name = api_id.split("/")[-1] if "/" in api_id else api_id + + if api_obj.get("tier") == "Premium" or any(kw in conn_name.lower() for kw in PREMIUM_KEYWORDS): + premium_connectors_found.add(conn_name) + self.logger.info(f" [!] Premium connector found: {conn_name} in flow {name}") + if "custom" in api_id.lower() or api_obj.get("type") == "Microsoft.PowerApps/apis/custom": + custom_connectors_found.add(conn_name) + self.logger.info(f" [!] Custom connector found: {conn_name} in flow {name}") + + actions_str = json.dumps(props) + has_nested_loops = actions_str.count('"type": "Foreach"') > 0 or actions_str.count('"type": "Until"') > 0 + has_multi_approvals = "shared_approvals" in actions_str or "Approval" in actions_str + has_advanced_expressions = "@" in actions_str and any(exp in actions_str for exp in ["concat(", "split(", "base64("]) + + if has_nested_loops or has_multi_approvals or has_advanced_expressions: + self.logger.info(f" [!] Complex logic detected in Cloud Flow: {name}") + reasons = [] + if has_nested_loops: reasons.append("Nested Loops") + if has_multi_approvals: reasons.append("Multi Approvals") + if has_advanced_expressions: reasons.append("Advanced Expressions") + + flow_dict = { + "Environment": env_display, "Name": name, "Type": "Cloud Flow", "Tier": tier, + "Active": "Yes" if is_active else "No", + "Reason": ", ".join(reasons) + } + with complex_flows_lock: + with open(complex_logic_flows_path, 'a', encoding='utf-8') as cf_f: + cf_f.write(json.dumps(flow_dict) + '\n') + + del flow_detail + del res + + # ========================================== + # 2. FETCH DESKTOP FLOWS + # ========================================== + instance_url = env_props.get("linkedEnvironmentMetadata", {}).get("instanceApiUrl") + + if instance_url: + dv_url = instance_url.rstrip("/") + try: + dv_token = self._get_access_token(f"{dv_url}/.default") + if not dv_token: + self.logger.warning(f" [X] Failed to acquire token for Dataverse instance {dv_url}") + continue + + headers_dv = { + "Authorization": f"Bearer {dv_token}", + "Accept": "application/json", + "OData-MaxVersion": "4.0", "OData-Version": "4.0" + } + + dv_api_url = f"{dv_url}/api/data/v9.2/workflows?$filter=category eq 6&$select=name,clientdata,ismanaged,statecode,_ownerid_value&$expand=ownerid" + desktop_flows = self.fetch_all_pages(dv_api_url, headers_dv, context_name="Dataverse Desktop Flows API") + + counts["Desktop Flows"] += len(desktop_flows) + active_desktop_flows = [f for f in desktop_flows if f.get("statecode") == 1] + active_counts["Desktop Flows"] += len(active_desktop_flows) + + self.logger.info(f" -> Desktop Flows: {len(desktop_flows)} total found, {len(active_desktop_flows)} are active.") + + for flow in desktop_flows: + name = flow.get("name", "Unnamed Desktop Flow") + is_managed = flow.get("ismanaged", False) + owner_name = flow.get("ownerid", {}).get("fullname", "Unknown / System") + statecode = flow.get("statecode") + is_active = (statecode == 1) + + if is_managed or "system" in owner_name.lower() or not is_default: + tier_counts["Enterprise/Departmental"] += 1 + tier = "Enterprise" + if is_active: + active_tier_counts["Enterprise/Departmental"] += 1 + else: + tier_counts["Personal Productivity"] += 1 + tier = "Personal" + if is_active: + active_tier_counts["Personal Productivity"] += 1 + + client_data_str = flow.get("clientdata", "") + if client_data_str: + try: + has_nested_loops = client_data_str.lower().count("foreach") > 1 + has_multi_approvals = client_data_str.count("shared_approvals") > 1 + has_advanced_expressions = "@" in client_data_str and any(exp in client_data_str for exp in ["concat(", "split(", "base64("]) + + if has_nested_loops or has_multi_approvals or has_advanced_expressions: + self.logger.info(f" [!] Complex logic detected in Desktop Flow: {name}") + reasons = [] + if has_nested_loops: reasons.append("Nested Loops") + if has_multi_approvals: reasons.append("Multi Approvals") + if has_advanced_expressions: reasons.append("Advanced Expressions") + + flow_dict = { + "Environment": env_display, "Name": name, "Type": "Desktop Flow", "Tier": tier, + "Active": "Yes" if is_active else "No", + "Reason": ", ".join(reasons) + } + with complex_flows_lock: + with open(complex_logic_flows_path, 'a', encoding='utf-8') as cf_f: + cf_f.write(json.dumps(flow_dict) + '\n') + except Exception: + pass + except Exception as e: + self.logger.error(f" [X] Failed to authenticate against Dataverse instance {dv_url}: {e}") + + complex_active_count = 0 + complex_inactive_count = 0 + with open(complex_logic_flows_path, 'r', encoding='utf-8') as f_cf: + for line in f_cf: + if json.loads(line).get("Active") == "Yes": + complex_active_count += 1 + else: + complex_inactive_count += 1 + + results = { + "total_environments": len(environments), + "counts": counts, + "active_counts": active_counts, + "tier_counts": tier_counts, + "active_tier_counts": active_tier_counts, + "premium_connectors": list(premium_connectors_found), + "custom_connectors": list(custom_connectors_found), + "complex_logic_flows_path": complex_logic_flows_path, + "complex_active_count": complex_active_count, + "complex_inactive_count": complex_inactive_count + } + + self.logger.info("Step End: Analysis complete.") + self.logger.info("Main Process End: Power Automate Telemetry scan finished.") + return results + + +# ================================================================================= +# MODULAR UI COMPONENT +# ================================================================================= + +class PowerAutomateUsageFrame(ctk.CTkFrame): + """Self-contained component wrapping Power Automate UI and export controls.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None # 'loading', 'success', 'error', None + self.last_complex_flows = [] + self.last_results = {} + + self.build_ui() + + def build_ui(self): + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + pa_header = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + pa_header.pack(fill="x", pady=(0, 10)) + ctk.CTkLabel(pa_header, text="Power Automate", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(side="left") + + self.btn_export_pa = ctk.CTkButton( + pa_header, text="Export Complex Flows", width=160, height=32, corner_radius=16, + font=FONT_BODY_BOLD, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self.export_complex_flows, state="disabled" + ) + self.btn_export_pa.pack(side="right") + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + # Height control slider for Power Automate Chart (packed above the chart dynamically) + self.pa_height_var = ctk.DoubleVar(value=400) + self.pa_slider_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + + self.slider_pa_height = ctk.CTkSlider( + self.pa_slider_frame, from_=200, to=800, number_of_steps=60, + variable=self.pa_height_var, width=120, height=16, + command=self._on_pa_height_slider_change + ) + self.slider_pa_height.pack(side="right") + + self.lbl_pa_height = ctk.CTkLabel(self.pa_slider_frame, text="Height: 400px", font=FONT_BODY_SMALL, text_color=COLOR_TEXT_SUB) + self.lbl_pa_height.pack(side="right", padx=(0, 10)) + + self.pa_chart_container = ctk.CTkFrame( + self.inner_pad, fg_color=COLOR_SURFACE, + border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8, + height=400 + ) + self.pa_chart_container.pack_propagate(False) + + self.reset_view() + + def reset_view(self): + self.pack_forget() + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + self.pa_slider_frame.pack_forget() + self.pa_chart_container.pack_forget() + self.btn_export_pa.configure(state="disabled") + self.last_complex_flows = [] + self.last_results = {} + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + for w in self.pa_chart_container.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=20) + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + display_msg = "Management Link / Flow read permissions required." + + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 10)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + usage_logger.info("Power Automate trigger_fetch called. Spawning background worker thread...") + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=10) + self.grid_frame.pack_forget() + self.pa_slider_frame.pack_forget() + self.pa_chart_container.pack_forget() + + self._set_state_loading("Scanning Power Automate flows...") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str): + if self.semaphore: + self.semaphore.acquire() + try: + scanner = PowerAutomateScanner(tenant, client_id, client_secret) + results = scanner.scan_flows() + usage_logger.info("Successfully completed Power Automate scan.") + self.after(0, self._render_success, results) + except Exception as e: + usage_logger.error("Exception caught in PowerAutomateUsage worker.", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, results: dict): + self.last_results = results + self.state_frame.pack_forget() + for w in self.grid_frame.winfo_children(): + w.destroy() + + self.grid_frame.pack(fill="x", expand=True) + + if not results: + empty_cell = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + empty_cell.pack(fill="x", expand=True, pady=15) + ctk.CTkLabel(empty_cell, text="No Power Automate data found.", text_color=COLOR_TEXT_SUB).pack() + self.status = "success" + self.on_status_change() + return + + total_envs = results.get("total_environments", 0) + counts = results.get("counts", {}) + active_counts = results.get("active_counts", {}) + tier_counts = results.get("tier_counts", {}) + active_tier_counts = results.get("active_tier_counts", {}) + premium_conns = results.get("premium_connectors", []) + custom_conns = results.get("custom_connectors", []) + complex_flows = results.get("complex_logic_flows", []) + + total_flows = counts.get("Cloud Flows", 0) + counts.get("Desktop Flows", 0) + + summary_frame = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_OUTLINE_LIGHT, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + summary_frame.pack(fill="x", pady=20) + + for i in range(2): + summary_frame.grid_columnconfigure(i, weight=1) + + headers_pa = ["Metric", "Value"] + for col_idx, head_text in enumerate(headers_pa): + cell = ctk.CTkFrame(summary_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + prem_str = ", ".join(premium_conns) if premium_conns else "0" + cust_str = ", ".join(custom_conns) if custom_conns else "0" + + mapping = [ + ("Total Environments Scanned", total_envs), + ("Total Flows (Active + Inactive)", total_flows), + ("Premium Connectors In Use", prem_str), + ("Custom Connectors In Use", cust_str), + ] + + r_idx = 1 + for label, val in mapping: + bg_style = COLOR_SURFACE if r_idx % 2 == 0 else COLOR_SURFACE_VARIANT + + c0 = ctk.CTkFrame(summary_frame, fg_color=bg_style, corner_radius=0) + c0.grid(row=r_idx, column=0, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c0, text=label, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="nw") + + c1 = ctk.CTkFrame(summary_frame, fg_color=bg_style, corner_radius=0) + c1.grid(row=r_idx, column=1, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c1, text=str(val), font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN, wraplength=400).pack(padx=10, pady=6, anchor="nw") + + r_idx += 1 + + complex_flows_path = results.get("complex_logic_flows_path") + self.last_complex_flows_path = complex_flows_path + if complex_flows_path and os.path.exists(complex_flows_path) and os.path.getsize(complex_flows_path) > 0: + self.btn_export_pa.configure(state="normal") + else: + self.btn_export_pa.configure(state="disabled") + + if total_flows > 0: + self.pa_slider_frame.pack(fill="x", pady=(10, 0)) + self.pa_chart_container.pack(fill="x", pady=(5, 20)) + for w in self.pa_chart_container.winfo_children(): + w.destroy() + + if not MATPLOTLIB_AVAILABLE: + ctk.CTkLabel(self.pa_chart_container, text="Matplotlib is required to render charts.\nPlease install it using 'pip install matplotlib'.", text_color=COLOR_ERROR).pack(pady=15) + else: + try: + fig = Figure(figsize=(10, 4), dpi=100) + ax = fig.add_subplot(111) + fig.patch.set_facecolor(COLOR_SURFACE) + ax.set_facecolor(COLOR_SURFACE) + + categories = ['Cloud Flows', 'Desktop Flows', 'Personal Flows', 'Enterprise Flows', 'Complex Flows'] + + c_total = counts.get("Cloud Flows", 0) + c_active = active_counts.get("Cloud Flows", 0) + c_inactive = c_total - c_active + + d_total = counts.get("Desktop Flows", 0) + d_active = active_counts.get("Desktop Flows", 0) + d_inactive = d_total - d_active + + p_total = tier_counts.get("Personal Productivity", 0) + p_active = active_tier_counts.get("Personal Productivity", 0) + p_inactive = p_total - p_active + + e_total = tier_counts.get("Enterprise/Departmental", 0) + e_active = active_tier_counts.get("Enterprise/Departmental", 0) + e_inactive = e_total - e_active + + complex_active = results.get("complex_active_count", 0) + complex_inactive = results.get("complex_inactive_count", 0) + + actives = [c_active, d_active, p_active, e_active, complex_active] + inactives = [c_inactive, d_inactive, p_inactive, e_inactive, complex_inactive] + + x = range(len(categories)) + width = 0.15 + + color_active = COLOR_PRIMARY + color_inactive = COLOR_TONAL_BG + + rects1 = ax.bar(x, actives, width, label='Active', color=color_active) + rects2 = ax.bar([i + width for i in x], inactives, width, label='Inactive', color=color_inactive) + + ax.set_ylabel('Count', color=COLOR_TEXT_MAIN, fontsize=10, fontweight='bold') + ax.set_title('Power Automate Flows Breakdown', color=COLOR_TEXT_MAIN, fontsize=12, fontweight='bold') + ax.set_xticks([i + width/2 for i in x]) + ax.set_xticklabels(categories, color=COLOR_TEXT_MAIN, fontsize=10, fontweight='bold') + ax.legend(facecolor=COLOR_SURFACE, edgecolor=COLOR_OUTLINE_LIGHT, labelcolor=COLOR_TEXT_MAIN, prop={'weight':'bold', 'size':9}) + + ax.bar_label(rects1, padding=3, color=COLOR_TEXT_MAIN, fontsize=9, fontweight='bold') + ax.bar_label(rects2, padding=3, color=COLOR_TEXT_MAIN, fontsize=9, fontweight='bold') + + for spine in ax.spines.values(): + spine.set_color(COLOR_OUTLINE_LIGHT) + + ax.tick_params(axis='y', colors=COLOR_TEXT_MAIN, labelsize=9) + for label in ax.get_yticklabels(): + label.set_fontweight('bold') + + max_val = max(max(actives), max(inactives)) + ax.set_ylim(0, max(max_val + 3, int(max_val * 1.3))) + + fig.tight_layout() + + canvas = FigureCanvasTkAgg(fig, master=self.pa_chart_container) + canvas.draw() + canvas.get_tk_widget().pack(fill="both", expand=True, padx=20, pady=10) + + except Exception as e: + usage_logger.error(f"Error drawing Power Automate charts: {e}", exc_info=True) + + self.status = "success" + self.on_status_change() + + def _render_error(self, err_msg): + usage_logger.warning(f"Power Automate fetch failed: {err_msg}") + self._set_state_error(err_msg) + self.status = "error" + self.on_status_change() + + def export_complex_flows(self): + usage_logger.info("Exporting complex flows to local spreadsheet requested.") + if not hasattr(self, "last_complex_flows_path") or not self.last_complex_flows_path: + return + + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + f = filedialog.asksaveasfilename( + initialfile=f"complex_flows_{ts}.csv", + defaultextension=".csv", + filetypes=[("CSV Spreadsheet", "*.csv")] + ) + + if not f: + return + + headers = ["Environment", "Name", "Type", "Tier", "Active", "Reason"] + + try: + import pandas as pd + df_iter = pd.read_json(self.last_complex_flows_path, lines=True, chunksize=1000) + for i, chunk in enumerate(df_iter): + chunk = chunk[headers] + chunk.to_csv(f, mode='a' if i > 0 else 'w', header=(i == 0), index=False, encoding='utf-8') + usage_logger.info("Complex flows exported successfully.") + messagebox.showinfo("Export Successful", f"Complex flows successfully saved to:\n{f}", parent=self) + except Exception as e: + usage_logger.error("Failed writing export spreadsheet to disk.", exc_info=True) + messagebox.showerror("Export Error", f"Failed to save file:\n{e}", parent=self) + + def _on_pa_height_slider_change(self, val): + height_val = int(val) + self.lbl_pa_height.configure(text=f"Height: {height_val}px") + self.pa_chart_container.configure(height=height_val) diff --git a/telemetry/sharepoint_onedrive_usage.py b/telemetry/sharepoint_onedrive_usage.py new file mode 100644 index 00000000..e87390d2 --- /dev/null +++ b/telemetry/sharepoint_onedrive_usage.py @@ -0,0 +1,537 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Modular SharePoint and OneDrive usage telemetry scanners and visual interfaces.""" + +import os +import pandas as pd +import logging +import threading +import customtkinter as ctk + +# Import unified core service layer +from core.graph.client import GraphClient +from core.graph.reports import ReportsService + +# Bind to the async logger initialized in m365_telemetry.py +usage_logger = logging.getLogger("M365TelemetryAsyncLogger") + +# ================================================================================= +# CONSTANTS & STYLES (Imported from shared styles) +# ================================================================================= +from telemetry.styles import * + +# ================================================================================= +# PIPELINE UTILITIES +# ================================================================================= + +def format_bytes(num_bytes: int) -> str: + """Formats raw byte values into highly readable string equivalents (e.g., GB, TB).""" + if num_bytes is None: + return "0.00 Bytes" + + for unit in ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB']: + if num_bytes < 1024.0: + return f"{num_bytes:,.2f} {unit}" + num_bytes /= 1024.0 + return f"{num_bytes:,.2f} EB" + +def parse_sharepoint_csv(filepath): + """Streams the SharePoint Site Usage Detail CSV and aggregates metrics in chunks.""" + usage_logger.info(f"Processing SharePoint Site Usage file: {os.path.basename(filepath)}") + if not os.path.exists(filepath): + usage_logger.error(f"Error: Could not find SharePoint report {filepath}") + raise FileNotFoundError(f"SharePoint Site Usage report not found.") + + headers = pd.read_csv(filepath, nrows=0).columns.tolist() + expected = ["Is Deleted", "Storage Used (Byte)", "File Count", "Active File Count"] + cols = [c for c in expected if c in headers] + + total_sites = 0 + total_storage = 0 + total_files = 0 + active_files = 0 + + for chunk in pd.read_csv(filepath, usecols=cols, chunksize=10000, encoding="utf-8-sig"): + if "Is Deleted" in chunk.columns: + mask = chunk["Is Deleted"].astype(str).str.strip().str.upper() != "TRUE" + active_chunk = chunk[mask] + else: + active_chunk = chunk + + total_sites += len(active_chunk) + if "Storage Used (Byte)" in active_chunk.columns: + total_storage += int(pd.to_numeric(active_chunk["Storage Used (Byte)"], errors='coerce').fillna(0).sum()) + if "File Count" in active_chunk.columns: + total_files += int(pd.to_numeric(active_chunk["File Count"], errors='coerce').fillna(0).sum()) + if "Active File Count" in active_chunk.columns: + active_files += int(pd.to_numeric(active_chunk["Active File Count"], errors='coerce').fillna(0).sum()) + + usage_logger.info(f"SharePoint parsing complete: sites={total_sites}, storage={format_bytes(total_storage)}, files={total_files}, active_files={active_files}") + return { + "total_sites": total_sites, + "total_storage_bytes": total_storage, + "total_storage_formatted": format_bytes(total_storage), + "total_files": total_files, + "active_files": active_files, + "active_files_pct": (active_files / total_files * 100) if total_files > 0 else 0.0 + } + +def parse_onedrive_csv(filepath): + """Streams the OneDrive Account Usage Detail CSV and aggregates metrics in chunks.""" + usage_logger.info(f"Processing OneDrive Account Usage file: {os.path.basename(filepath)}") + if not os.path.exists(filepath): + usage_logger.error(f"Error: Could not find OneDrive report {filepath}") + raise FileNotFoundError(f"OneDrive Account Usage report not found.") + + headers = pd.read_csv(filepath, nrows=0).columns.tolist() + expected = ["Is Deleted", "Storage Used (Byte)", "File Count", "Active File Count"] + cols = [c for c in expected if c in headers] + + total_accounts = 0 + total_storage = 0 + total_files = 0 + active_files = 0 + + for chunk in pd.read_csv(filepath, usecols=cols, chunksize=10000, encoding="utf-8-sig"): + if "Is Deleted" in chunk.columns: + mask = chunk["Is Deleted"].astype(str).str.strip().str.upper() != "TRUE" + active_chunk = chunk[mask] + else: + active_chunk = chunk + + total_accounts += len(active_chunk) + if "Storage Used (Byte)" in active_chunk.columns: + total_storage += int(pd.to_numeric(active_chunk["Storage Used (Byte)"], errors='coerce').fillna(0).sum()) + if "File Count" in active_chunk.columns: + total_files += int(pd.to_numeric(active_chunk["File Count"], errors='coerce').fillna(0).sum()) + if "Active File Count" in active_chunk.columns: + active_files += int(pd.to_numeric(active_chunk["Active File Count"], errors='coerce').fillna(0).sum()) + + usage_logger.info(f"OneDrive parsing complete: accounts={total_accounts}, storage={format_bytes(total_storage)}, files={total_files}, active_files={active_files}") + return { + "total_accounts": total_accounts, + "total_storage_bytes": total_storage, + "total_storage_formatted": format_bytes(total_storage), + "total_files": total_files, + "active_files": active_files, + "active_files_pct": (active_files / total_files * 100) if total_files > 0 else 0.0 + } + +def parse_onedrive_activity_csv(filepath): + """Streams the OneDrive Activity User Detail CSV and aggregates active sync client users in chunks.""" + usage_logger.info(f"Processing OneDrive Activity User Detail file: {os.path.basename(filepath)}") + if not os.path.exists(filepath): + usage_logger.error(f"Error: Could not find OneDrive Activity report {filepath}") + raise FileNotFoundError(f"OneDrive Activity User Detail report not found.") + + headers = pd.read_csv(filepath, nrows=0).columns.tolist() + expected = ["Is Deleted", "Synced File Count"] + cols = [c for c in expected if c in headers] + + sync_users = 0 + + for chunk in pd.read_csv(filepath, usecols=cols, chunksize=10000, encoding="utf-8-sig"): + if "Is Deleted" in chunk.columns: + mask = chunk["Is Deleted"].astype(str).str.strip().str.upper() != "TRUE" + active_chunk = chunk[mask] + else: + active_chunk = chunk + + if "Synced File Count" in active_chunk.columns: + synced_series = pd.to_numeric(active_chunk["Synced File Count"], errors='coerce').fillna(0) + sync_users += int((synced_series > 0).sum()) + + usage_logger.info(f"OneDrive Activity parsing complete: sync_users={sync_users}") + return { + "sync_users": sync_users + } + +def parse_onenote_users_csv(filepath): + """Streams the M365 App User Detail CSV and counts unique active OneNote users in chunks.""" + usage_logger.info(f"Processing OneNote Users file: {os.path.basename(filepath)}") + if not os.path.exists(filepath): + usage_logger.error(f"Error: Could not find M365 App User Detail report {filepath}") + raise FileNotFoundError(f"M365 App User Detail report not found.") + + headers = pd.read_csv(filepath, nrows=0).columns.tolist() + expected = ["Is Deleted", "OneNote"] + cols = [c for c in expected if c in headers] + + onenote_users = 0 + + for chunk in pd.read_csv(filepath, usecols=cols, chunksize=10000, encoding="utf-8-sig"): + if "Is Deleted" in chunk.columns: + mask = chunk["Is Deleted"].astype(str).str.strip().str.upper() != "TRUE" + active_chunk = chunk[mask] + else: + active_chunk = chunk + + if "OneNote" in active_chunk.columns: + onenote_series = active_chunk["OneNote"].astype(str).str.strip().str.lower() + onenote_users += int(onenote_series.isin(["yes", "true"]).sum()) + + usage_logger.info(f"OneNote Users parsing complete: onenote_users={onenote_users}") + return { + "onenote_users": onenote_users + } + +# ================================================================================= +# INDEPENDENT SCANNING PIPELINES +# ================================================================================= + +def run_sharepoint_pipeline(client_id, client_secret, tenant_id) -> dict: + """Pipeline specifically for SharePoint telemetry data collection.""" + usage_logger.info("Starting SharePoint Telemetry Pipeline...") + + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=5, + backoff=2 + ) + client.authenticate() + service = ReportsService(client) + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + reports_dir = os.path.join(script_dir, "reports", f"{tenant_id}_{client_id}") + + service.download_sharepoint_details(reports_dir) + usage_logger.info("SharePoint Site Usage CSV download completed. Initiating parser...") + client.close() + + sp_data = parse_sharepoint_csv(os.path.join(reports_dir, "SharePointSiteUsageDetail(180d).csv")) + usage_logger.info("SharePoint Telemetry Pipeline completed successfully.") + return sp_data + +def run_onedrive_pipeline(client_id, client_secret, tenant_id) -> dict: + """Pipeline specifically for OneDrive telemetry data collection.""" + usage_logger.info("Starting OneDrive Telemetry Pipeline...") + + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=2, + retries=5, + backoff=2 + ) + client.authenticate() + service = ReportsService(client) + + script_dir = os.path.dirname(os.path.abspath(__file__)) if '__file__' in globals() else os.getcwd() + reports_dir = os.path.join(script_dir, "reports", f"{tenant_id}_{client_id}") + + service.download_onedrive_details(reports_dir) + usage_logger.info("OneDrive account, activity, and OneNote CSV downloads completed. Initiating parsers...") + client.close() + + od_data = parse_onedrive_csv(os.path.join(reports_dir, "OneDriveUsageAccountDetail(180d).csv")) + od_act_data = parse_onedrive_activity_csv(os.path.join(reports_dir, "OneDriveActivityUserDetail(180d).csv")) + onenote_data = parse_onenote_users_csv(os.path.join(reports_dir, "M365AppUserDetail_sp_od(180d).csv")) + + # Merge active sync client user data into od_data + od_data["sync_users"] = od_act_data["sync_users"] + od_data["sync_users_pct"] = (od_act_data["sync_users"] / od_data["total_accounts"] * 100) if od_data["total_accounts"] > 0 else 0.0 + + # Merge OneNote user data + od_data["onenote_users"] = onenote_data["onenote_users"] + + usage_logger.info("OneDrive Telemetry Pipeline completed successfully.") + return od_data + +# ================================================================================= +# MODULAR UI COMPONENTS +# ================================================================================= + +class SharePointUsageFrame(ctk.CTkFrame): + """Self-contained customtkinter component wrapping SharePoint Telemetry UI.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None # 'loading', 'success', 'error', None + + self.build_ui() + + def build_ui(self): + """Creates card container for the tab.""" + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + ctk.CTkLabel(self.inner_pad, text="SharePoint Site Usage (180 Days)", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(anchor="w", pady=(0, 10)) + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_OUTLINE_LIGHT, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.reset_view() + + def reset_view(self): + """Resets and hides grids.""" + self.pack_forget() + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=20) + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + display_msg = "Reports telemetry permission required.\nPlease grant the 'Reports.Read.All' application permission to your App Registration in Microsoft Entra ID." + + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 10)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + """Triggers parallel fetches inside isolated background threads.""" + usage_logger.info("SharePoint Site Usage trigger_fetch called. Spawning background worker thread...") + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=(20, 10)) + self.grid_frame.pack_forget() + + self._set_state_loading("Downloading and parsing SharePoint Site Usage reports...") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str): + usage_logger.info("Executing thread: _execute_sharepoint_worker") + if self.semaphore: + self.semaphore.acquire() + try: + data = run_sharepoint_pipeline(client_id, client_secret, tenant) + usage_logger.info("Successfully completed SharePoint telemetry data fetch.") + self.after(0, self._render_success, data) + except Exception as e: + usage_logger.error("Exception caught in SharePoint worker.", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, data: dict): + self.last_data = data + usage_logger.info("SharePoint Site Usage data successfully retrieved. Rendering UI grid.") + self.state_frame.pack_forget() + for w in self.grid_frame.winfo_children(): + w.destroy() + + self.grid_frame.pack(fill="x", expand=True) + + self.grid_frame.grid_columnconfigure(0, weight=3) + self.grid_frame.grid_columnconfigure(1, weight=2) + + headers_sp = ["SharePoint Metric Description", "Value / Measurement"] + for col_idx, head_text in enumerate(headers_sp): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + rows_data = [ + ("Total Sites Count", f"{data.get('total_sites', 0):,} Sites"), + ("Total Storage Used", data.get("total_storage_formatted", "0.00 Bytes")), + ("Total Files Stored", f"{data.get('total_files', 0):,} Files"), + ("Active Files Count", f"{data.get('active_files', 0):,} Files ({data.get('active_files_pct', 0.0):.1f}%)") + ] + + for r_idx, (metric_name, val) in enumerate(rows_data, start=1): + bg_style = COLOR_SURFACE if r_idx % 2 == 0 else COLOR_SURFACE_VARIANT + + c0 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c0.grid(row=r_idx, column=0, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c0, text=metric_name, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + c1 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c1.grid(row=r_idx, column=1, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c1, text=val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + self.status = "success" + self.on_status_change() + + def _render_error(self, err_msg): + usage_logger.warning(f"SharePoint Site Usage fetch failed: {err_msg}") + self._set_state_error(err_msg) + self.status = "error" + self.on_status_change() + + +class OneDriveUsageFrame(ctk.CTkFrame): + """Self-contained customtkinter component wrapping OneDrive Telemetry UI.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.status = None # 'loading', 'success', 'error', None + + self.build_ui() + + def build_ui(self): + """Creates card container for the tab.""" + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + ctk.CTkLabel(self.inner_pad, text="OneDrive Usage (180 Days)", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(anchor="w", pady=(0, 10)) + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_OUTLINE_LIGHT, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.reset_view() + + def reset_view(self): + """Resets and hides grids.""" + self.pack_forget() + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=20) + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + display_msg = "Reports telemetry permission required.\nPlease grant the 'Reports.Read.All' application permission to your App Registration in Microsoft Entra ID." + + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 10)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients[0], secrets[0]) + + def trigger_fetch(self, tenant, client_id, client_secret): + """Triggers parallel fetches inside isolated background threads.""" + usage_logger.info("OneDrive Usage trigger_fetch called. Spawning background worker thread...") + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=(20, 10)) + self.grid_frame.pack_forget() + + self._set_state_loading("Downloading and parsing OneDrive reports...") + + threading.Thread( + target=self._execute_worker, + args=(tenant, client_id, client_secret), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, client_id: str, client_secret: str): + usage_logger.info("Executing thread: _execute_onedrive_worker") + if self.semaphore: + self.semaphore.acquire() + try: + data = run_onedrive_pipeline(client_id, client_secret, tenant) + usage_logger.info("Successfully completed OneDrive telemetry data fetch.") + self.after(0, self._render_success, data) + except Exception as e: + usage_logger.error("Exception caught in OneDrive worker.", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, data: dict): + self.last_data = data + usage_logger.info("OneDrive Usage data successfully retrieved. Rendering UI grid.") + self.state_frame.pack_forget() + for w in self.grid_frame.winfo_children(): + w.destroy() + + self.grid_frame.pack(fill="x", expand=True) + + self.grid_frame.grid_columnconfigure(0, weight=3) + self.grid_frame.grid_columnconfigure(1, weight=2) + + headers_od = ["OneDrive Metric Description", "Value / Measurement"] + for col_idx, head_text in enumerate(headers_od): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + rows_data = [ + ("Total Accounts Count", f"{data.get('total_accounts', 0):,} Accounts"), + ("Total Storage Used", data.get("total_storage_formatted", "0.00 Bytes")), + ("Total Files Stored", f"{data.get('total_files', 0):,} Files"), + ("Active Files Count", f"{data.get('active_files', 0):,} Files ({data.get('active_files_pct', 0.0):.1f}%)"), + ("Users with Synced Files", f"{data.get('sync_users', 0):,} Users ({data.get('sync_users_pct', 0.0):.1f}%)"), + ("OneNote Active Users", f"{data.get('onenote_users', 0):,} Users") + ] + + for r_idx, (metric_name, val) in enumerate(rows_data, start=1): + bg_style = COLOR_SURFACE if r_idx % 2 == 0 else COLOR_SURFACE_VARIANT + + c0 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c0.grid(row=r_idx, column=0, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c0, text=metric_name, font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + c1 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c1.grid(row=r_idx, column=1, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c1, text=val, font=FONT_BODY_MEDIUM, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=6, anchor="w") + + self.status = "success" + self.on_status_change() + + def _render_error(self, err_msg): + usage_logger.warning(f"OneDrive Usage fetch failed: {err_msg}") + self._set_state_error(err_msg) + self.status = "error" + self.on_status_change() diff --git a/telemetry/styles.py b/telemetry/styles.py new file mode 100644 index 00000000..074befce --- /dev/null +++ b/telemetry/styles.py @@ -0,0 +1,37 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared Material 3 style constants and typography tokens for the Telemetry module.""" + +COLOR_PRIMARY = "#1A73E8" +COLOR_PRIMARY_HOVER = "#1557B0" +COLOR_SURFACE = "#FFFFFF" +COLOR_SURFACE_VARIANT = "#F1F3F4" +COLOR_SURFACE_HOVER = "#EFF6FF" +COLOR_TONAL_BG = "#E8F0FE" +COLOR_TONAL_TEXT = "#1A73E8" +COLOR_TEXT_MAIN = "#202124" +COLOR_TEXT_SUB = "#5F6368" +COLOR_OUTLINE = "#80868B" +COLOR_OUTLINE_LIGHT = "#E8EAED" +COLOR_SUCCESS = "#137333" +COLOR_ERROR = "#C5221F" +COLOR_SECONDARY_HOVER = "#F1F3F4" + +FONT_HEADER_MEDIUM = ("Segoe UI", 22, "bold") +FONT_HEADER_SMALL = ("Segoe UI", 16, "bold") +FONT_BODY_BOLD = ("Segoe UI", 12, "bold") +FONT_BODY_MEDIUM = ("Segoe UI", 12, "normal") +FONT_BODY_SMALL = ("Segoe UI", 10, "normal") + diff --git a/telemetry/subscribed_skus.py b/telemetry/subscribed_skus.py new file mode 100644 index 00000000..f9b6d8ea --- /dev/null +++ b/telemetry/subscribed_skus.py @@ -0,0 +1,302 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Modular Subscribed SKUs Inventory Summary telemetry scanners and visual interfaces.""" + +import os +import logging +import threading +import webbrowser +import pandas as pd +from datetime import datetime +from tkinter import filedialog, messagebox +from typing import Any, Dict, List, Optional +import customtkinter as ctk + +# Import unified core service layer +from core.graph.client import GraphClient +from core.graph.directory import DirectoryService + +# Bind to the async logger initialized in m365_telemetry.py +usage_logger = logging.getLogger("M365TelemetryAsyncLogger") + +# ================================================================================= +# CONSTANTS & STYLES (Imported from shared styles) +# ================================================================================= +from telemetry.styles import * + +class SubscribedSKUsFrame(ctk.CTkFrame): + """Self-contained customtkinter component wrapping Subscribed SKUs Inventory Summary UI.""" + + def __init__(self, master, log_callback, credentials_callback, status_change_callback, retries_var=None, backoff_var=None, **kwargs): + self.semaphore = kwargs.pop("concurrency_semaphore", None) + super().__init__(master, fg_color=COLOR_SURFACE, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=12, **kwargs) + self.log_msg = log_callback + self.get_credentials = credentials_callback + self.on_status_change = status_change_callback + self.retries = retries_var + self.backoff = backoff_var + self.status = None # 'loading', 'success', 'error', None + self.last_licenses_items = [] + + self.build_ui() + + def build_ui(self): + """Creates card container for the tab.""" + self.pack(fill="x", expand=True, pady=10) + + self.inner_pad = ctk.CTkFrame(self, fg_color="transparent") + self.inner_pad.pack(fill="both", expand=True, padx=20, pady=20) + + self.lic_header = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.lic_header.pack(fill="x", pady=(0, 10)) + ctk.CTkLabel(self.lic_header, text="Subscribed SKUs", font=FONT_HEADER_SMALL, text_color=COLOR_TEXT_MAIN).pack(side="left") + + self.lic_reference_link = ctk.CTkLabel( + self.lic_header, + text="Service Plan Reference ↗", + font=FONT_BODY_BOLD, + text_color=COLOR_PRIMARY, + cursor="hand2" + ) + self.lic_reference_link.pack(side="left", padx=(15, 0)) + self.lic_reference_link.bind("", lambda e: webbrowser.open("https://learn.microsoft.com/en-us/entra/identity/users/licensing-service-plan-reference")) + self.lic_reference_link.bind("", lambda e: self.lic_reference_link.configure(text_color=COLOR_PRIMARY_HOVER)) + self.lic_reference_link.bind("", lambda e: self.lic_reference_link.configure(text_color=COLOR_PRIMARY)) + + ctk.CTkLabel(self.lic_header, text="* To view specific services offered, export the spreadsheet.", font=FONT_BODY_SMALL, text_color=COLOR_TEXT_SUB).pack(side="left", padx=(10, 0)) + + self.btn_export_lic = ctk.CTkButton( + self.lic_header, text="Export Spreadsheet", width=140, height=32, corner_radius=16, + font=FONT_BODY_BOLD, fg_color="transparent", border_width=1, border_color=COLOR_OUTLINE, + text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER, + command=self.export_licenses_spreadsheet, state="disabled" + ) + self.btn_export_lic.pack(side="right") + + self.state_frame = ctk.CTkFrame(self.inner_pad, fg_color="transparent") + self.grid_frame = ctk.CTkFrame(self.inner_pad, fg_color=COLOR_OUTLINE_LIGHT, border_color=COLOR_OUTLINE_LIGHT, border_width=1, corner_radius=8) + + self.reset_view() + + def reset_view(self): + """Resets and hides grids.""" + self.pack_forget() + self.state_frame.pack_forget() + self.grid_frame.pack_forget() + self.btn_export_lic.configure(state="disabled") + self.last_licenses_items = [] + + for w in self.state_frame.winfo_children(): + w.destroy() + for w in self.grid_frame.winfo_children(): + w.destroy() + + def _set_state_loading(self, msg="Loading..."): + for w in self.state_frame.winfo_children(): + w.destroy() + ctk.CTkLabel(self.state_frame, text=f"⏳ {msg}", text_color=COLOR_TEXT_SUB, font=FONT_BODY_MEDIUM).pack(pady=20) + self.state_frame.pack(fill="x", expand=True) + + def _set_state_error(self, error_msg): + for w in self.state_frame.winfo_children(): + w.destroy() + + display_msg = error_msg + if "401" in error_msg or "403" in error_msg or "unauthorized" in error_msg.lower() or "forbidden" in error_msg.lower(): + display_msg = "Directory/Organization read permission required.\nPlease grant the 'Organization.Read.All' or 'Directory.Read.All' permission to your App Registration in Entra ID." + + ctk.CTkLabel(self.state_frame, text=f"✖ {display_msg}", text_color=COLOR_ERROR, font=FONT_BODY_MEDIUM, justify="center").pack(pady=(20, 10)) + ctk.CTkButton(self.state_frame, text="Try Again", command=self._retry_fetch, width=120, fg_color="transparent", border_width=1, text_color=COLOR_PRIMARY, hover_color=COLOR_SECONDARY_HOVER).pack(pady=(0, 20)) + self.state_frame.pack(fill="x", expand=True) + + def _retry_fetch(self): + tenant, clients, secrets = self.get_credentials() + if tenant: + self.trigger_fetch(tenant, clients, secrets) + + def trigger_fetch(self, tenant, clients, secrets): + """Triggers SKU fetch inside background thread.""" + usage_logger.info("SKU fetch trigger_fetch called. Spawning background worker thread...") + self.status = "loading" + self.on_status_change() + + self.pack(fill="x", expand=True, pady=10) + self.grid_frame.pack_forget() + + self._set_state_loading("Fetching subscribed SKUs...") + + retries_val = self.retries.get() if self.retries else 5 + backoff_val = self.backoff.get() if self.backoff else 2 + + threading.Thread( + target=self._execute_worker, + args=(tenant, clients, secrets, retries_val, backoff_val), + daemon=True + ).start() + + def _execute_worker(self, tenant: str, clients: List[str], secrets: List[str], retries_val: int, backoff_val: int): + usage_logger.info("Executing thread: _execute_sku_worker") + if self.semaphore: + self.semaphore.acquire() + try: + client_id = clients[0] + client_secret = secrets[0] + + self.log_msg(f"Authenticating app {client_id[:5]}...") + + client = GraphClient( + tenant_id=tenant, + client_ids=client_id, + client_secrets=client_secret, + concurrency=1, + retries=retries_val, + backoff=backoff_val + ) + + required_scopes = ["Organization.Read.All", "Directory.Read.All"] + client.authenticate(required_scopes=required_scopes) + + self.log_msg("Querying Graph API endpoint for SKUs...") + dir_service = DirectoryService(client) + sku_data = dir_service.get_subscribed_skus() + client.close() + + usage_logger.info("Successfully fetched SKU data.") + self.after(0, self._render_success, sku_data) + except Exception as e: + usage_logger.error("Exception caught in SubscribedSKUs worker.", exc_info=True) + self.after(0, self._render_error, str(e)) + finally: + if self.semaphore: + self.semaphore.release() + + def _render_success(self, sku_dict: Dict[str, Any]): + usage_logger.info("Executing UI render for SKU table.") + self.state_frame.pack_forget() + for w in self.grid_frame.winfo_children(): + w.destroy() + + self.grid_frame.pack(fill="x", expand=True) + + self.grid_frame.grid_columnconfigure(0, weight=2) + self.grid_frame.grid_columnconfigure(1, weight=1) + self.grid_frame.grid_columnconfigure(2, weight=1) + + headers = ["SKU Part Number", "Units", "Consumed Units"] + for col_idx, head_text in enumerate(headers): + cell = ctk.CTkFrame(self.grid_frame, fg_color=COLOR_TONAL_BG, corner_radius=0) + cell.grid(row=0, column=col_idx, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(cell, text=head_text, font=FONT_BODY_BOLD, text_color=COLOR_TONAL_TEXT).pack(padx=10, pady=8, anchor="w") + + items = sku_dict.get("value", []) + if not items: + empty_cell = ctk.CTkFrame(self.grid_frame, fg_color="transparent") + empty_cell.grid(row=1, column=0, columnspan=3, sticky="nsew", pady=15) + ctk.CTkLabel(empty_cell, text="No subscribed product configurations found in scope.", text_color=COLOR_TEXT_SUB).pack() + else: + items.sort(key=lambda x: len(x.get("servicePlans", [])), reverse=True) + self.last_licenses_items = items + self.btn_export_lic.configure(state="normal") + + current_row = 1 + for item_idx, item in enumerate(items): + bg_style = COLOR_SURFACE if item_idx % 2 == 0 else COLOR_SURFACE_VARIANT + + c0 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c0.grid(row=current_row, column=0, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c0, text=item.get("skuPartNumber", "UNKNOWN_SKU"), font=FONT_BODY_BOLD, text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=8, anchor="nw") + + c1 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c1.grid(row=current_row, column=1, sticky="nsew", padx=0, pady=(0, 1)) + prepaid = item.get("prepaidUnits", {}) + p_str = f"Enabled: {prepaid.get('enabled', 0):,}" + if prepaid.get('warning', 0) > 0: p_str += f"\nWarn: {prepaid.get('warning'):,}" + if prepaid.get('suspended', 0) > 0: p_str += f"\nSusp: {prepaid.get('suspended'):,}" + ctk.CTkLabel(c1, text=p_str, text_color=COLOR_TEXT_MAIN, justify="left").pack(padx=10, pady=8, anchor="nw") + + c2 = ctk.CTkFrame(self.grid_frame, fg_color=bg_style, corner_radius=0) + c2.grid(row=current_row, column=2, sticky="nsew", padx=0, pady=(0, 1)) + ctk.CTkLabel(c2, text=f"{item.get('consumedUnits', 0):,}", text_color=COLOR_TEXT_MAIN).pack(padx=10, pady=8, anchor="nw") + + current_row += 1 + + self.status = "success" + self.on_status_change() + + def _render_error(self, err_msg): + usage_logger.warning(f"Rendering SKU table error state: {err_msg}") + self._set_state_error(err_msg) + self.btn_export_lic.configure(state="disabled") + self.status = "error" + self.on_status_change() + + def export_licenses_spreadsheet(self): + """Exports the SKUs inventory to a CSV formatted to mimic merged cells.""" + usage_logger.info("Exporting licenses to local spreadsheet requested.") + if not self.last_licenses_items: + usage_logger.warning("Export aborted: No cached license items available to export.") + return + + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + f = filedialog.asksaveasfilename( + initialfile=f"licenses_inventory_{ts}.csv", + defaultextension=".csv", + filetypes=[("CSV Spreadsheet", "*.csv")] + ) + + if not f: + usage_logger.info("Export aborted by user (dialog cancelled).") + return + + usage_logger.info(f"Target export path established: {f}") + headers = ["SKU Part Number", "Units", "Consumed Units", "Included Service Plans", "Applies To"] + rows = [] + + for item in self.last_licenses_items: + sku_name = item.get("skuPartNumber", "UNKNOWN_SKU") + prepaid = item.get("prepaidUnits", {}) + enabled_units = prepaid.get("enabled", 0) + warn_units = prepaid.get("warning", 0) + susp_units = prepaid.get("suspended", 0) + + prepaid_str = f"Enabled: {enabled_units:,}" + if warn_units > 0: prepaid_str += f"\nWarn: {warn_units:,}" + if susp_units > 0: prepaid_str += f"\nSusp: {susp_units:,}" + consumed_str = f"{item.get('consumedUnits', 0):,}" + + plans = item.get("servicePlans", []) + + if not plans: + rows.append([sku_name, prepaid_str, consumed_str, "None designated.", "-"]) + else: + for idx, p in enumerate(plans): + p_name = p.get("servicePlanName", "UnnamedPlan") + p_scope = p.get("appliesTo", "Unknown") + if idx == 0: + rows.append([sku_name, prepaid_str, consumed_str, p_name, p_scope]) + else: + rows.append(["", "", "", p_name, p_scope]) + + try: + chunk_size = 1000 + for i in range(0, len(rows), chunk_size): + chunk = rows[i:i + chunk_size] + df = pd.DataFrame(chunk, columns=headers) + df.to_csv(f, mode='a' if i > 0 else 'w', header=(i == 0), index=False, encoding='utf-8') + usage_logger.info("Spreadsheet exported successfully.") + messagebox.showinfo("Export Successful", f"Spreadsheet successfully saved to:\n{f}", parent=self) + except Exception as e: + usage_logger.error("Failed writing export spreadsheet to disk.", exc_info=True) + messagebox.showerror("Export Error", f"Failed to save file:\n{e}", parent=self) diff --git a/telemetry/user_persona_analysis.py b/telemetry/user_persona_analysis.py new file mode 100644 index 00000000..32c8913e --- /dev/null +++ b/telemetry/user_persona_analysis.py @@ -0,0 +1,586 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Modular User Persona Analysis dataset extraction and processing pipelines.""" + +import os +import json +import sqlite3 +import logging +import pandas as pd +from google import genai +from google.genai import types +from sklearn.preprocessing import StandardScaler +from sklearn.cluster import KMeans +from sklearn.metrics import silhouette_score +from core.graph.client import GraphClient +from core.graph.reports import ReportsService + +# Bind to the async logger initialized in m365_telemetry.py +logger = logging.getLogger("M365TelemetryAsyncLogger.UserPersonaAnalysis") + + +def run_user_persona_pipeline(tenant_id: str, client_id: str, client_secret: str, gemini_api_key: str, output_csv_path: str, strategy: str = "heuristic", reports_dir: str = None, status_callback=None) -> dict: + """Runs the full pipeline to download M365 reports, generate user activity dataset, + and query Gemini to build and classify user personas using the selected strategy. + + Returns a dictionary containing the personas definition and user assignments. + """ + if not reports_dir: + reports_dir = os.path.join("telemetry", "reports", f"{tenant_id}_{client_id}") + + os.makedirs(reports_dir, exist_ok=True) + + # 1. Download reports & aggregate data + if status_callback: + status_callback("Fetching Reports") + + # Authenticate and initialize Graph clients + logger.info("Initializing Graph connection for User Persona Analysis...") + client = GraphClient( + tenant_id=tenant_id, + client_ids=client_id, + client_secrets=client_secret, + concurrency=2, + retries=5, + backoff=2 + ) + client.authenticate() + reports_service = ReportsService(client) + + reports = [ + ("https://graph.microsoft.com/v1.0/reports/getOffice365ActiveUserDetail(period='D180')", "Office365ActiveUserDetail(180d).csv"), + ("https://graph.microsoft.com/v1.0/reports/getEmailActivityUserDetail(period='D180')", "EmailActivityUserDetail(180d).csv"), + ("https://graph.microsoft.com/v1.0/reports/getTeamsUserActivityUserDetail(period='D180')", "TeamsUserActivityUserDetail(180d).csv"), + ("https://graph.microsoft.com/v1.0/reports/getOneDriveUsageAccountDetail(period='D180')", "OneDriveUsageAccountDetail(180d).csv"), + ("https://graph.microsoft.com/v1.0/reports/getSharePointActivityUserDetail(period='D180')", "SharePointActivityUserDetail(180d).csv") + ] + + logger.info("Downloading reports in batch concurrently...") + reports_service.download_reports_batch(reports, reports_dir) + + if status_callback: + status_callback("Aggregating Data") + + logger.info("Reports download complete. Starting dataset generation...") + generate_user_activity_dataset(reports_dir, output_csv_path) + + # 2. Execute Selected Strategy + if strategy == "kmeans": + if status_callback: + status_callback("Clustering the data") + + logger.info("Executing K-Means clustering strategy...") + df_users = pd.read_csv(output_csv_path) + + # Run clustering + df_clustered, cluster_summary, optimal_k = perform_kmeans_clustering(df_users) + + if status_callback: + status_callback("Generating insights using Gemini") + + # Call Gemini to summarize centroids + personas_response = generate_kmeans_personas_gemini(gemini_api_key, cluster_summary, optimal_k) + + personas_list = personas_response.get("personas", []) + + # Format assigned personas ID as cluster_X + df_clustered['Assigned_Persona_ID'] = 'cluster_' + df_clustered['Cluster_ID'].astype(str) + + # Map persona IDs to Titles + persona_titles = {p["id"]: p["title"] for p in personas_list} + df_clustered['Assigned_Persona_Title'] = df_clustered['Assigned_Persona_ID'].map(lambda p_id: persona_titles.get(p_id, "Unknown")) + + # Save the updated clustered dataset + df_clustered.to_csv(output_csv_path, index=False) + + return { + "personas": personas_list, + "dataset_path": output_csv_path + } + + else: + # Heuristic Strategy + if status_callback: + status_callback("Generating insights using Gemini") + + logger.info("Calling Gemini API to define personas...") + personas_response = generate_personas_from_dataset(gemini_api_key, output_csv_path) + + # Classify users in Python + logger.info("Classifying users to generated personas in Python...") + df_users = pd.read_csv(output_csv_path) + personas_list = personas_response.get("personas", []) + + assigned_persona_ids = classify_users_to_personas(df_users, personas_list) + df_users['Assigned_Persona_ID'] = assigned_persona_ids + + # Map persona IDs to Titles + persona_titles = {p["id"]: p["title"] for p in personas_list} + df_users['Assigned_Persona_Title'] = df_users['Assigned_Persona_ID'].map(lambda p_id: persona_titles.get(p_id, "Unknown")) + + # Save the updated dataset containing persona assignments + df_users.to_csv(output_csv_path, index=False) + + return { + "personas": personas_list, + "dataset_path": output_csv_path + } + + +def generate_user_activity_dataset(reports_dir: str, output_csv_path: str) -> None: + """Streams and joins downloaded user reports via a temporary SQLite DB to preserve memory.""" + temp_db_path = os.path.join(reports_dir, "temp_persona_data.db") + + # Clean up any leftover DB from previous runs + if os.path.exists(temp_db_path): + try: + os.remove(temp_db_path) + except Exception: + pass + + conn = sqlite3.connect(temp_db_path) + cursor = conn.cursor() + + # Define mapping of CSV files to database tables + report_files = { + "Office365ActiveUserDetail(180d).csv": "office365_active_users", + "EmailActivityUserDetail(180d).csv": "email_activity", + "TeamsUserActivityUserDetail(180d).csv": "teams_activity", + "OneDriveUsageAccountDetail(180d).csv": "onedrive_usage", + "SharePointActivityUserDetail(180d).csv": "sharepoint_activity" + } + + try: + # 1. Load CSVs into SQLite tables in chunks + for csv_name, table_name in report_files.items(): + csv_path = os.path.join(reports_dir, csv_name) + if not os.path.exists(csv_path): + logger.warning(f"Report file {csv_name} not found. Skipping table {table_name}.") + continue + + logger.info(f"Importing {csv_name} into SQLite table {table_name}...") + + for chunk in pd.read_csv(csv_path, chunksize=20000, encoding="utf-8-sig"): + # Clean column headers + chunk.columns = chunk.columns.str.strip() + chunk.to_sql(table_name, conn, if_exists="append", index=False) + + # Ensure base table exists + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='office365_active_users';") + if not cursor.fetchone(): + raise FileNotFoundError("Base table 'office365_active_users' could not be loaded because the required CSV is missing.") + + # 2. Create indices to optimize joins + logger.info("Creating database indices for efficient join operations...") + index_queries = [ + "CREATE INDEX IF NOT EXISTS idx_base_upn ON office365_active_users (`User Principal Name`);", + "CREATE INDEX IF NOT EXISTS idx_email_upn ON email_activity (`User Principal Name`);", + "CREATE INDEX IF NOT EXISTS idx_teams_upn ON teams_activity (`User Principal Name`);", + "CREATE INDEX IF NOT EXISTS idx_onedrive_upn ON onedrive_usage (`Owner Principal Name`);", + "CREATE INDEX IF NOT EXISTS idx_sp_upn ON sharepoint_activity (`User Principal Name`);" + ] + for query in index_queries: + try: + cursor.execute(query) + except sqlite3.OperationalError as e: + logger.warning(f"Failed to create index. Query: {query}. Error: {e}") + + conn.commit() + + # 3. Query combined fields and stream the results to CSV + join_query = """ + SELECT + b.`User Principal Name` AS `User Principal Name`, + b.`Has Exchange License` AS `Has Exchange License`, + b.`Has OneDrive License` AS `Has OneDrive License`, + b.`Has SharePoint License` AS `Has SharePoint License`, + b.`Has Teams License` AS `Has Teams License`, + e.`Send Count` AS `Email_Sends`, + e.`Meeting Created Count` AS `Meetings_Organized_Via_Email`, + t.`Private Chat Message Count` AS `Teams_Private_Chats`, + t.`Meetings Attended Count` AS `Teams_Meetings_Attended`, + o.`Active File Count` AS `OneDrive_Active_Files`, + o.`Storage Used (Byte)` AS `OneDrive_Storage_Bytes`, + s.`Viewed Or Edited File Count` AS `SharePoint_Files_Edited`, + s.`Shared Internally File Count` AS `SharePoint_Shared_Internally`, + s.`Shared Externally File Count` AS `SharePoint_Shared_Externally` + FROM office365_active_users b + LEFT JOIN email_activity e ON b.`User Principal Name` = e.`User Principal Name` + LEFT JOIN teams_activity t ON b.`User Principal Name` = t.`User Principal Name` + LEFT JOIN onedrive_usage o ON b.`User Principal Name` = o.`Owner Principal Name` + LEFT JOIN sharepoint_activity s ON b.`User Principal Name` = s.`User Principal Name` + """ + + logger.info("Executing database join and streaming dataset to CSV...") + + # Ensure parent output directory exists + output_dir = os.path.dirname(output_csv_path) + if output_dir: + os.makedirs(output_dir, exist_ok=True) + + # Delete output file if it already exists from a previous run + if os.path.exists(output_csv_path): + os.remove(output_csv_path) + + first_chunk = True + + def determine_access_profile(row): + def is_truthy(val): + if val is None: + return False + if isinstance(val, bool): + return val + s = str(val).strip().lower() + return s in ['true', '1', '1.0', 'yes'] + + apps = [] + if is_truthy(row.get('Has Exchange License')): apps.append('Email') + if is_truthy(row.get('Has Teams License')): apps.append('Teams') + if is_truthy(row.get('Has OneDrive License')) or is_truthy(row.get('Has SharePoint License')): apps.append('Files') + + if len(apps) == 3: return "Full Suite Access" + if len(apps) == 0: return "Restricted / Unlicensed" + return "Partial Access: " + " + ".join(apps) + + + # Read chunks from SQLite database + for chunk in pd.read_sql_query(join_query, conn, chunksize=20000): + # Clean whitespaces in column values or headers if any + chunk.columns = chunk.columns.str.strip() + + # Calculate App Access Profile + chunk['App_Access_Profile'] = chunk.apply(determine_access_profile, axis=1) + + # Drop raw boolean license columns + license_cols = [c for c in chunk.columns if c.startswith('Has ')] + chunk = chunk.drop(columns=license_cols, errors='ignore') + + # Convert OneDrive Bytes to GB + if 'OneDrive_Storage_Bytes' in chunk.columns: + bytes_col = pd.to_numeric(chunk['OneDrive_Storage_Bytes'], errors='coerce').fillna(0) + chunk['OneDrive_Storage_GB'] = (bytes_col / (1024 ** 3)).round(2) + chunk = chunk.drop(columns=['OneDrive_Storage_Bytes']) + + # Fill missing values (NaN) with 0 for numerical columns + numerical_cols = chunk.select_dtypes(include=['float64', 'int64']).columns + chunk[numerical_cols] = chunk[numerical_cols].fillna(0) + + # Reorder columns: put App_Access_Profile next to User Principal Name + cols = chunk.columns.tolist() + if 'App_Access_Profile' in cols and 'User Principal Name' in cols: + cols.insert(1, cols.pop(cols.index('App_Access_Profile'))) + chunk = chunk[cols] + + # Append chunk to output CSV + chunk.to_csv(output_csv_path, mode='a', index=False, header=first_chunk) + first_chunk = False + + finally: + conn.close() + # Delete temporary SQLite DB file + if os.path.exists(temp_db_path): + try: + os.remove(temp_db_path) + logger.info("Temporary database removed successfully.") + except Exception as e: + logger.warning(f"Could not remove temporary database file: {e}") + + +def generate_personas_from_dataset(api_key: str, dataset_path: str) -> dict: + """Invokes Gemini SDK to analyze M365 telemetry data and identify user personas.""" + if not os.path.exists(dataset_path): + raise FileNotFoundError(f"Dataset not found at: {dataset_path}") + + # Read the entire CSV data + with open(dataset_path, "r", encoding="utf-8") as f: + csv_data = f.read() + + prompt = f"""You are an expert M365 telemetry and data analyst. +Below is the complete M365 user activity dataset (180 days) for all users in a tenant. + +CSV Dataset: +{csv_data} + +Based on this M365 dataset, analyze the behavioral patterns and define 3 to 5 distinct user personas that represent the typical usage behavior of the users in this tenant. + +It is CRITICAL to include an 'Inactive or Idle Accounts' persona (with all metrics set to 'low') if there are accounts in the dataset with zero or near-zero activity across all telemetry counts. These represent inactive, unlicensed, or archive accounts. + +For each persona, output: +1. A unique ID (e.g., "email_collaborator"). +2. A title/headline (e.g., "Email Collaborator"). +3. A representative emoji/icon (e.g., "📧"). +4. A brief description of this persona. +5. 2 to 4 behavior patterns (bullet points explaining their characteristics). +6. A metric profile specifying the relative usage of each behavior category as "high", "medium", or "low". + +Output your response strictly as a JSON object matching this schema: +{{ + "personas": [ + {{ + "id": "email_collaborator", + "title": "Email Collaborator", + "emoji": "📧", + "description": "Users who heavily rely on Email communications...", + "behavior_patterns": [ + "High volume of email sends", + "Active calendar organizer" + ], + "metrics": {{ + "email_sends": "high", + "meetings_organized": "medium", + "teams_chats": "low", + "teams_meetings": "low", + "onedrive_files": "low", + "sharepoint_edits": "low", + "sharepoint_shared_internal": "low", + "sharepoint_shared_external": "low", + "onedrive_storage": "low" + }} + }} + ] +}} + +Do not include any other markdown formatting, code block fences, or text outside the JSON.""" + + # Initialize the genai Client directly with the API key + client = genai.Client(api_key=api_key) + + logger.info("Calling Gemini generation with SDK model: gemini-flash-latest...") + response = client.models.generate_content( + model="gemini-flash-latest", + contents=prompt, + config=types.GenerateContentConfig( + response_mime_type="application/json" + ) + ) + + try: + candidate_text = response.text + return json.loads(candidate_text) + except (KeyError, IndexError, ValueError) as e: + logger.error(f"Failed to parse Gemini response: {e}. Raw response: {response}") + raise ValueError(f"Failed to parse Gemini response: {e}") + + +def classify_users_to_personas(df_users: pd.DataFrame, personas_data: list) -> list: + """Clusts and maps M365 users to defined personas in Python using distance calculations.""" + cols_to_normalize = [ + 'Email_Sends', 'Meetings_Organized_Via_Email', + 'Teams_Private_Chats', 'Teams_Meetings_Attended', + 'OneDrive_Active_Files', 'SharePoint_Files_Edited', + 'SharePoint_Shared_Internally', 'SharePoint_Shared_Externally', + 'OneDrive_Storage_GB' + ] + + # Calculate ranks (0 to 1) for each column in the dataframe + ranks_df = pd.DataFrame(index=df_users.index) + for col in cols_to_normalize: + if col in df_users.columns: + if df_users[col].max() == df_users[col].min(): + # If all values are 0, rank is 0.0, else 0.5 + ranks_df[col] = 0.0 if df_users[col].max() == 0 else 0.5 + else: + # Calculate percentile rank + ranks_df[col] = df_users[col].rank(pct=True) + # Force absolute 0 values to rank 0.0 so they don't get inflated due to ties at 0 + ranks_df.loc[df_users[col] == 0, col] = 0.0 + else: + ranks_df[col] = 0.0 + + # Map profile levels to numeric values + level_map = { + "high": 0.8, + "medium": 0.4, + "low": 0.1 + } + + # Define normalized mapping of persona metrics to df columns + norm_metric_keys = { + "emailsends": "Email_Sends", + "meetingsorganized": "Meetings_Organized_Via_Email", + "teamschats": "Teams_Private_Chats", + "teamsmeetings": "Teams_Meetings_Attended", + "onedrive_files": "OneDrive_Active_Files", # backup keys + "onedrivefiles": "OneDrive_Active_Files", + "sharepointedits": "SharePoint_Files_Edited", + "sharepointsharedinternal": "SharePoint_Shared_Internally", + "sharepointsharedexternal": "SharePoint_Shared_Externally", + "onedrivestorage": "OneDrive_Storage_GB" + } + + user_personas = [] + + for idx, row in df_users.iterrows(): + best_persona_id = None + min_distance = float('inf') + + rank_row = ranks_df.loc[idx] + + for persona in personas_data: + p_metrics = persona.get("metrics", {}) + + # Normalize keys returned by LLM to lowercase without spaces/underscores/hyphens + norm_p_metrics = {} + for k, v in p_metrics.items(): + norm_k = str(k).lower().replace("_", "").replace("-", "").replace(" ", "") + norm_p_metrics[norm_k] = v + + distance = 0.0 + + for p_key, col_name in norm_metric_keys.items(): + # Get the value from the normalized metrics dict + p_val_str = str(norm_p_metrics.get(p_key, "low")).lower() + p_val = level_map.get(p_val_str, 0.1) + user_val = rank_row[col_name] + distance += (user_val - p_val) ** 2 + + if distance < min_distance: + min_distance = distance + best_persona_id = persona["id"] + + user_personas.append(best_persona_id) + + return user_personas + + +def perform_kmeans_clustering(df: pd.DataFrame, max_clusters: int = 6) -> tuple[pd.DataFrame, pd.DataFrame, int]: + """Runs K-Means clustering on employee usage telemetry. + Automatically determines the optimal number of clusters (2 to max_clusters) using Silhouette scores. + """ + logger.info(f"Running K-Means clustering dynamic selection (Max: {max_clusters})...") + + # Select numerical columns for clustering + cols_to_cluster = [ + 'Email_Sends', 'Meetings_Organized_Via_Email', + 'Teams_Private_Chats', 'Teams_Meetings_Attended', + 'OneDrive_Active_Files', 'SharePoint_Files_Edited', + 'SharePoint_Shared_Internally', 'SharePoint_Shared_Externally', + 'OneDrive_Storage_GB' + ] + + # Filter columns that actually exist in the dataframe + cluster_cols = [c for c in cols_to_cluster if c in df.columns] + if not cluster_cols: + raise ValueError("No numerical columns found in the dataset to perform clustering.") + + X = df[cluster_cols].copy() + + # Fill NaN values with 0 + X = X.fillna(0.0) + + # Scale the metrics + scaler = StandardScaler() + X_scaled = scaler.fit_transform(X) + + best_k = 2 + best_score = -1 + best_kmeans = None + + # Test k up to min(max_clusters, number of rows - 1) + limit_k = min(max_clusters, len(df) - 1) + if limit_k < 2: + kmeans = KMeans(n_clusters=len(df), random_state=42, n_init=10) + labels = kmeans.fit_predict(X_scaled) + df['Cluster_ID'] = labels + + centroid_cols = ['Cluster_ID'] + cluster_cols + cluster_summary = df[centroid_cols].groupby('Cluster_ID').mean().round(2) + cluster_summary['User_Count'] = df['Cluster_ID'].value_counts() + return df, cluster_summary, len(df) + + for k in range(2, limit_k + 1): + kmeans = KMeans(n_clusters=k, random_state=42, n_init=10) + labels = kmeans.fit_predict(X_scaled) + + score = silhouette_score(X_scaled, labels) + msg = f" Tested k={k} -> Silhouette Score: {score:.4f}" + logger.info(msg) + + if score > best_score: + best_score = score + best_k = k + best_kmeans = kmeans + + optimal_msg = f"Optimal cluster count determined: {best_k} (Silhouette Score: {best_score:.4f})" + logger.info(optimal_msg) + + df['Cluster_ID'] = best_kmeans.labels_ + + # Calculate centroids + centroid_cols = ['Cluster_ID'] + cluster_cols + cluster_summary = df[centroid_cols].groupby('Cluster_ID').mean().round(2) + cluster_summary['User_Count'] = df['Cluster_ID'].value_counts() + + return df, cluster_summary, best_k + + +def generate_kmeans_personas_gemini(api_key: str, cluster_summary: pd.DataFrame, optimal_k: int) -> dict: + """Sends K-Means centroids and user count data to Gemini to define visual personas.""" + cluster_data_text = cluster_summary.to_string() + + prompt = f"""You are an expert IT Operations and SaaS Licensing Analyst. +I have run a K-Means clustering algorithm on employee Microsoft 365 usage telemetry spanning a 180-day period. + +The algorithm automatically determined that the workforce divides best into {optimal_k} distinct employee clusters. +Here is the centroid data (average behavior metrics and user count) for each cluster: + +Centroid Summary: +{cluster_data_text} + +Analyze the centroid values for each cluster in great detail and translate them into a distinct, creative user persona. +For each cluster, define: +1. An ID matching "cluster_[Cluster_ID]" (e.g. "cluster_0", "cluster_1"). +2. A creative, professional persona name (title). +3. A representative emoji character. +4. A behavioral summary description (2-3 sentences). +5. 2 to 4 behavior patterns (bullet points explaining their characteristics based on the metric values). + +Output your response strictly as a JSON object matching this schema: +{{ + "personas": [ + {{ + "id": "cluster_0", + "title": "Collaborative Power User", + "emoji": "🚀", + "description": "Users in this cluster are highly active in Teams and email communication...", + "behavior_patterns": [ + "Very high Teams chat volume", + "Active meeting participant" + ] + }} + ] +}} + +Do not include any other markdown formatting, code block fences, or text outside the JSON.""" + + # Initialize client directly with the API key + client = genai.Client(api_key=api_key) + + logger.info("Calling K-Means Gemini summarization with SDK model: gemini-flash-latest...") + response = client.models.generate_content( + model="gemini-flash-latest", + contents=prompt, + config=types.GenerateContentConfig( + response_mime_type="application/json" + ) + ) + + try: + candidate_text = response.text + return json.loads(candidate_text) + except (KeyError, IndexError, ValueError) as e: + logger.error(f"Failed to parse Gemini response: {e}. Raw response: {response}") + raise ValueError(f"Failed to parse Gemini response: {e}") +